:W&Bá¾)éî§M 75ctabfrª; 9 /0.28.1+bc2cf24b1bd9fa3a1b6cbeb920087b832afc526f0.70.0X\GòÄ‚" q2sfj699dr2o 75ctabfrŠ  75ctabfrspatop" _wandb‚{}Bspatop_v6_blocks_outputjdp-spatop-train-v6-blocks-lqsdgŠ ÍšÅÓЋÄš1 (H¨¸h"3.12.10*0.28.1j linux-x86_64–ôcÒÝ -Linux-6.8.0-134-generic-x86_64-with-glibc2.36CPython 3.12.10 ÍšÅÓôÛŒ*-of*A/data/spatop/options_files/sweep_fixed_dp/config_gs8pex8v_v6.json*-l*/scratch/logs/*-n*spatop_v6_blocks_output2-m spanet.trainRdanielprimosch@gmail.comZ/scratch/logs/bdp-spatop-train-v6-blocks-lqsdgr/opt/conda/bin/pythonº gi04rfix29a1zqykn5fgh7ym23adhlgaèŒÛW2  requirements.txtPþÝ$P 75ctabfrZ? (H¨¸ (H¨¸h"3.12.10*0.28.1j linux-x86_64;~~ÉÇÒÁ€@ˆ@’NVIDIA A100 80GB PCIe˜¢ /€€€´ð7€àÏ’­ª€À½­ÝÂS NVIDIA A100 80GB PCIe€€€€À€6"Ampere*(GPU-1c79f124-b049-1e3d-7f6d-104357f4ddb6Ê13.2º gi04rfix29a1zqykn5fgh7ym23adhlga&¼HrR 75ctabfrZA (H¨¸ (H¨¸8h"3.12.10*0.28.1j linux-x86_645*ëX* 75ctabfrb trainer/global_step2…ÀŸ,U  75ctabfrZD (H¨¸ (H¨¸8h"3.12.10*0.28.1j linux-x86_64On]Ã/  75ctabfrb*"trainer/global_step2%ºK  75ctabfrj: ΚÅӈ臥*LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] PÖcÊ“  75ctabfrj  ΚÅÓð§úÚî â”â”â”â”┳â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”┓ ┃ ┃ Name ┃ Type ┃ Params ┃ Mode ┃ FLOPs ┃ ┡â”â”â”╇â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”┩ │ 0 │ embedding │ MultiInputVectorEmbed… │ 220 K │ train │ 0 │ │ 1 │ encoder │ JetEncoderWithPairwise │ 332 K │ train │ 0 │ │ 2 │ branch_decoders │ ModuleList │ 4.8 M │ train │ 0 │ │ 3 │ regression_decoder │ RegressionDecoder │ 0 │ train │ 0 │ │ 4 │ classification_decoder │ ClassificationDecoder │ 0 │ train │ 0 │ │ │ other params │ n/a │ 12 │ n/a │ n/a │ └───┴────────────────────────┴────────────────────────┴────────┴───────┴───────┘ ^^a\  75ctabfrjÊ ΚÅÓÀ–‚Ü·Trainable params: 5.3 M Non-trainable params: 50 Total params: 5.3 M Total estimated model params size (MB): 21.374 Modules in train mode: 2326 Modules in eval mode: 0 Total FLOPs: 0 ä!¼½ 75ctabfr*«  hidden_dim‚64  transformer_dim‚64  transformer_dim_scale‚2.0  initial_embedding_dim‚64  position_embedding_dim‚64  num_embedding_layers‚10  num_encoder_layers‚4 ! num_branch_embedding_layers‚3  num_branch_encoder_layers‚3  num_jet_embedding_layers‚0  num_jet_encoder_layers‚2  num_detector_layers‚2  num_regression_layers‚3  num_classification_layers‚3  split_symmetric_attention‚1  num_attention_heads‚8 ! transformer_activation‚"gelu"  skip_connections‚1 ( "initial_embedding_skip_connections‚1  linear_block_type‚"GRU"  transformer_type‚"Gated"  linear_activation‚"gelu"  normalization‚ "LayerNorm"  masking‚ "Filling" linear_prelu_activation‚true  use_pairwise_interactions‚1  num_pairwise_features‚4  pairwise_embedding_dim‚8  pairwise_input_source‚"all"  pairwise_block_embeddings‚1  pairwise_cross_type‚true  use_moe‚false  num_experts‚8  num_experts_per_tok‚2  moe_loss_scale‚0.01 K event_info_file‚7"/data/spatop/event_files/v11/tt_hadronic_v7_full.yaml" K training_file‚9"/data/spatop/delphes_v6/tt_hadronic_training2k_clean.h5"  validation_file‚""  testing_file‚""  normalize_features‚1  limit_to_num_jets‚0  balance_particles‚0  balance_jets‚0 ! balance_classifications‚false  partial_events‚1  dataset_limit‚1.0  dataset_randomization‚0  train_validation_split‚0.95  limit_val_batches‚1.0  batch_size‚512  num_dataloader_workers‚0  mask_sequence_vectors‚1  combine_pair_loss‚"min"  optimizer‚"AdamW" ' learning_rate‚0.0007344299982717679  focal_gamma‚0.0  combinatorial_scale‚0.0 # learning_rate_warmup_epochs‚1.0  learning_rate_cycles‚1  assignment_loss_scale‚1.0  detection_loss_scale‚1.0  kl_loss_scale‚0.0  regression_loss_scale‚0.0 ! classification_loss_scale‚0.0 ! use_mass_classification‚false  mass_classes‚[]  balance_losses‚true  l2_penalty‚0.0002  gradient_clip‚10.0  dropout‚0.2978723651141102 epochs‚10 num_gpu‚1  verbose_output‚false  usable_gpus‚""  trial_time‚"" $ trial_output_dir‚"./test_output"k×·ê 75ctabfrjØ ÏšÅÓÀÄõÈ/opt/conda/lib/python3.12/site-packages/pytorch_lightning/utilities/_pytree.py:21: `isinstance(treespec, LeafSpec)` is deprecated, use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead. ®2 M 75ctabfrj» ÏšÅÓȾ„«/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:434: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance. «ðë 75ctabfrjÙ КÅÓÐçó¶È/opt/conda/lib/python3.12/site-packages/pytorch_lightning/utilities/_pytree.py:21: `isinstance(treespec, LeafSpec)` is deprecated, use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead.  Ö+P 75ctabfrj¾ КÅÓзƒ·­/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:434: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance. ¬§s±‚(:† ÝšÅÓʳçƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚31.0 gpu.0.powerWatts‚125.652' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚41.884 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ï4¡7‚(:­  ÝšÅÓÔîù„ disk.drbd2118.in‚0 disk.drbd1931.out‚0 disk.rbd0.out‚0 disk.dm-0.in‚0 disk.drbd1828.out‚0 disk.dm-0.out‚ 3.6826171875$ disk./.usageGB‚587.2863006591797 disk.drbd1817.in‚0 disk.drbd1965.in‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd2041.out‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚0 disk.drbd1226.in‚0 disk.drbd2078.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 0.1181640625 disk.drbd2041.in‚0$ proc.memory.rssMB‚11253.82421875* disk./.usagePercent‚32.844124348176095 disk.sda1.in‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.drbd1285.in‚0$ memory_percent‚41.21118187904358) proc.memory.percent‚17.17197299003601 disk.drbd1552.out‚0 disk.sda.out‚ 0.5703125 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 cpu‚18.19622524645334 disk.drbd1289.out‚0 disk.drbd1356.in‚0 proc.cpu.threads‚29 disk.drbd1584.out‚0 disk.sda2.out‚ 0.5703125 disk.rbd1.in‚0 network.sent‚43177 disk.drbd1673.out‚0 disk.drbd1584.in‚0 disk.drbd1828.in‚0* proc.memory.availableMB‚38527.83984375 network.recv‚45193 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.drbd1938.out‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0! disk.nvme0n1.out‚ 3.6826171875 disk.drbd1356.out‚0 disk.sda.in‚0 disk.drbd1965.out‚0 disk.drbd1510.in‚0 disk.drbd1931.in‚0 disk.drbd2111.in‚0 disk.drbd1938.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0%¬Ä93b/ lr-AdamW/pg1"trainer/global_step2:JP-s`3b/ lr-AdamW/pg2"trainer/global_step2:JPTNÄ-) _wandbruntime‚23  _runtime‚23X%ò½ ' lr-AdamW/pg1‚1.2628371377799987e-06 ' lr-AdamW/pg2‚1.2628371377799987e-06 trainer/global_step‚49 % _timestamp‚1.7858102787595973e+09 _runtime‚ 23.949651901 _step‚0vöHÄ¿ ' lr-AdamW/pg1‚1.2628371377799987e-06 ' lr-AdamW/pg2‚1.2628371377799987e-06 trainer/global_step‚49 % _timestamp‚1.7858102787595973e+09 _runtime‚ 23.949651901 _step‚0¾Ÿcíibe BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated"trainer/global_step2:JPa¨„gbc @DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current"trainer/global_step2:JP\ØSpbl IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated"trainer/global_step2:JPùïr™gbc @DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated"trainer/global_step2:JPΉÙjbf CDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed"trainer/global_step2:JPû°®xbt QDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JPô3‡s bo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak"trainer/global_step2:JP¸ðÙ2n!bj GDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated"trainer/global_step2:JP![eW"bS 0DeviceStatsMonitor.on_train_batch_start/num_ooms"trainer/global_step2:JPég#bc @DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak"trainer/global_step2:JPÃ^Ïëk$bg DDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated"trainer/global_step2:JPÞÛÚ½j%bf CDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current"trainer/global_step2:JPT1q&bm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current"trainer/global_step2:JPä·ûef'bb ?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak"trainer/global_step2:JPB˜Á¤r(bn KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated"trainer/global_step2:JP “L˜f)bb ?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak"trainer/global_step2:JPf!õl*bh EDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current"trainer/global_step2:JP§=Ÿ2m+bi FDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak"trainer/global_step2:JPmj0e,ba >DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak"trainer/global_step2:JPnàì‘s-bo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated"trainer/global_step2:JPÌ*ïn.bj GDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed"trainer/global_step2:JP—×i/be BDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current"trainer/global_step2:JP¼È©n0bj GDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak"trainer/global_step2:JP?¬p1bl IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated"trainer/global_step2:JP:^±Ýt2bp MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed"trainer/global_step2:JP}]ùi3be BDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed"trainer/global_step2:JP×/æ*o4bk HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed"trainer/global_step2:JP°õ\žn5bj GDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak"trainer/global_step2:JPy ÕSi6be BDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak"trainer/global_step2:JPBéþcj7bf CDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated"trainer/global_step2:JPlÛad8b` =DeviceStatsMonitor.on_train_batch_start/segment.all.allocated"trainer/global_step2:JPt«—ƒa9b] :DeviceStatsMonitor.on_train_batch_start/active.all.current"trainer/global_step2:JP­ŠBþi:be BDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak"trainer/global_step2:JPUX—n;bj GDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed"trainer/global_step2:JP”ønpB(ž`=b\ 9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries"trainer/global_step2:JPÊ›a”q>bm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current"trainer/global_step2:JP²wÆh?bd ADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed"trainer/global_step2:JP¿vQkp@bl IDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current"trainer/global_step2:JP—&æcAb_ DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak"trainer/global_step2:JPµ›‘ÝqCbm JDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current"trainer/global_step2:JPè¿áblDbh EDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current"trainer/global_step2:JP½ÙjEbf CDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current"trainer/global_step2:JPžžksFbo LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated"trainer/global_step2:JPþ U©kGbg DDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current"trainer/global_step2:JP—Ì`hHbd ADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak"trainer/global_step2:JPËóÊÉkIbg DDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated"trainer/global_step2:JP‹Å”Î_Jb[ 8DeviceStatsMonitor.on_train_batch_start/active.all.freed"trainer/global_step2:JP°¾ÂgKbc @DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak"trainer/global_step2:JP(kZnLbj GDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak"trainer/global_step2:JP¡à»ÝkMbg DDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak"trainer/global_step2:JP^úþlNbh EDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated"trainer/global_step2:JPaÛb§sObo LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated"trainer/global_step2:JPˆöxÉdPb` =DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak"trainer/global_step2:JP¯³=OqQbm JDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current"trainer/global_step2:JP%o‘“rRbn KDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated"trainer/global_step2:JPíðqSbm JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated"trainer/global_step2:JPÐoSŽhTbd ADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed"trainer/global_step2:JPÀ$ünUbj GDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak"trainer/global_step2:JP"Þ•¥mhbi FDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak"trainer/global_step2:JP¾e–=iibe BDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current"trainer/global_step2:JP˜`$;bjb^ ;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak"trainer/global_step2:JPÓ$¸gkbc @DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed"trainer/global_step2:JPB¸s=olbk HDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current"trainer/global_step2:JP# eÖxmbt QDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JP¥ '._nb[ 8DeviceStatsMonitor.on_train_batch_start/num_device_alloc"trainer/global_step2:JPIÊ^obZ 7DeviceStatsMonitor.on_train_batch_start/num_device_free"trainer/global_step2:JP“ŒZipbe BDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current"trainer/global_step2:JPô¿†^qbZ 7DeviceStatsMonitor.on_train_batch_start/active.all.peak"trainer/global_step2:JPd8žgrbc @DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed"trainer/global_step2:JP 6º›_sb[ 8DeviceStatsMonitor.on_train_batch_start/segment.all.peak"trainer/global_step2:JPÞ2dôhtbd ADeviceStatsMonitor.on_train_batch_start/active.large_pool.current"trainer/global_step2:JPÑP Ãeuba >DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak"trainer/global_step2:JPëÌ—evba >DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed"trainer/global_step2:JP—‰^kwbg DDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak"trainer/global_step2:JPÙ(Jòoxbk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed"trainer/global_step2:JP={mybi FDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed"trainer/global_step2:JPœþ=¹tzbp MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP,À-?b{b^ ;DeviceStatsMonitor.on_train_batch_start/segment.all.current"trainer/global_step2:JP‘":Þn|bj GDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current"trainer/global_step2:JP.(Ær}bn KDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated"trainer/global_step2:JPXZªs~bo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak"trainer/global_step2:JPû=`b\ 9DeviceStatsMonitor.on_train_batch_start/segment.all.freed"trainer/global_step2:JPT£ h€bc @DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed"trainer/global_step2:JPfK#gbb ?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed"trainer/global_step2:JPœW÷m‚bh EDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed"trainer/global_step2:JPN”Qÿfƒba >DeviceStatsMonitor.on_train_batch_start/allocation.all.current"trainer/global_step2:JP§o„bj GDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed"trainer/global_step2:JP¬»‰^n…bi FDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak"trainer/global_step2:JP` Yk†bf CDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed"trainer/global_step2:JPtÌig‡bb ?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak"trainer/global_step2:JPrÜbˆb] :DeviceStatsMonitor.on_train_batch_start/num_oom_rejections"trainer/global_step2:JPØrˆÜl‰bg DDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated"trainer/global_step2:JPE¾ÐïjŠbe BDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current"trainer/global_step2:JP•9¾Qm‹bh EDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak"trainer/global_step2:JP¶µ>mŒbh EDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated"trainer/global_step2:JPªE9nbi FDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak"trainer/global_step2:JP ¿°oŽbj GDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current"trainer/global_step2:JPöƒú pbk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed"trainer/global_step2:JPÛn¡ibd ADeviceStatsMonitor.on_train_batch_start/oversize_segments.current"trainer/global_step2:JPqúoºm‘bh EDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed"trainer/global_step2:JPЩ2Å^’bY 6DeviceStatsMonitor.on_train_batch_start/max_split_size"trainer/global_step2:JP!Þÿt“bo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated"trainer/global_step2:JPÄ"ÉÍi”bd ADeviceStatsMonitor.on_train_batch_start/active.small_pool.current"trainer/global_step2:JP5Çu?.•) _wandbruntime‚23  _runtime‚23ãú&{|$–ÞN RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250522112 VEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 906560747520 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 862880830464 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940454912 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚199247 [JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 911253711360 RADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 885038633400 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚534307 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11814 O>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 906456278528 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 43592487424 WFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 911100230656 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 866180174848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚144579 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 23.954660067 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚336727 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 862863791104 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚144329 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚524651 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚514 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚343826 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 VEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 885141475884 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 862863791104 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 43592487424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 841570332832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 906560747520 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 43679917056 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚871036 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚336727 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 866205078528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚336729 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874880 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 43571143052 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250522112 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 862880830464 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 44920055808 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 841553293472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11814 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚336729 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚199245 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 RADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 906456278528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 GDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_púæB¶èool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 43679917056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚861378 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940454912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856J*n h'—âN SBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 906560747520 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 43679917056 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚871036 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚336727 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 866205078528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚336729 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874880 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 43571143052 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250522112 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 862880830464 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 44920055808 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 841553293472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11814 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚336729 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚199245 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 RADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 906456278528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 GDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 43679917056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚861378 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940454912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250522112 VEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 906560747520 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 862880830464 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940454912 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚199247 [JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 911253711360 RADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 885038633400 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚534307 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11814 O>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 906456278528 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 43592487424 WFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 911100230656 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 866180174848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚144579 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 23.954660067 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚336727 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 862863791104 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚144329 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚524651 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚514 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚343826 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 VEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 885141475884 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 862863791104 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 43592487424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 841570332832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656ì<+l˜bg DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak"trainer/global_step2:JP³äÎ3g™bb ?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current"trainer/global_step2:JP¤p7lqšbl IDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated"trainer/global_step2:JP‹¶›îm›bh EDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak"trainer/global_step2:JPÇå³²hœbc @DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current"trainer/global_step2:JPŒb°qbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated"trainer/global_step2:JP–Àͦkžbf CDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current"trainer/global_step2:JP§½lŸbg DDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed"trainer/global_step2:JP„*˜á\ bW 4DeviceStatsMonitor.on_train_batch_end/max_split_size"trainer/global_step2:JP­ûk¡bf CDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated"trainer/global_step2:JP›úk¢bf CDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current"trainer/global_step2:JPS™Ûˆm£bh EDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated"trainer/global_step2:JPb°K·i¤bd ADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed"trainer/global_step2:JPâe0hg¥bb ?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current"trainer/global_step2:JP+ÅƵr¦bm JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated"trainer/global_step2:JPq窔a§b\ 9DeviceStatsMonitor.on_train_batch_end/segment.all.current"trainer/global_step2:JPÆèxÂh¨bc @DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated"trainer/global_step2:JP<Ÿ¤äf©ba >DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed"trainer/global_step2:JPª^Ÿqªbl IDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated"trainer/global_step2:JP4^5`«b[ 8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections"trainer/global_step2:JP?h Ãj¬be BDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current"trainer/global_step2:JP™dêêk­bf CDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated"trainer/global_step2:JP bn f®ba >DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed"trainer/global_step2:JPT>kc¯b^ ;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak"trainer/global_step2:JPÃòK m°bh EDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current"trainer/global_step2:JPlÿ­Fi±bd ADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current"trainer/global_step2:JPJ¿$Gh²bc @DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak"trainer/global_step2:JPLìà¥p³bk HDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current"trainer/global_step2:JP==(Yj´be BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated"trainer/global_step2:JP0Ûðoµbj GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated"trainer/global_step2:JP"f" n¶bi FDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current"trainer/global_step2:JP9ÙS&r·bm JDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak"trainer/global_step2:JPõñv·n¸bi FDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed"trainer/global_step2:JPÉúÈh^¹bY 6DeviceStatsMonitor.on_train_batch_end/segment.all.peak"trainer/global_step2:JP|&*áiºbd ADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated"trainer/global_step2:JPКd»b_ DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak"trainer/global_step2:JP’ €mÄbh EDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak"trainer/global_step2:JPÇpUÄ`Åb[ 8DeviceStatsMonitor.on_train_batch_end/active.all.current"trainer/global_step2:JP APÐ^ÆbY 6DeviceStatsMonitor.on_train_batch_end/active.all.freed"trainer/global_step2:JPý’»jÇbe BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated"trainer/global_step2:JP·LÌoÈbj GDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current"trainer/global_step2:JP¢âÁ%]ÉbX 5DeviceStatsMonitor.on_train_batch_end/num_device_free"trainer/global_step2:JP“ÔpÐbk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current"trainer/global_step2:JPO ómÑbh EDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed"trainer/global_step2:JP àolÒbg DDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak"trainer/global_step2:JP…uÿškÓbf CDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak"trainer/global_step2:JPòT2hÔbc @DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current"trainer/global_step2:JPXšLÇpÕbk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current"trainer/global_step2:JP" C„uÖbp MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current"trainer/global_step2:JPËÆËp_×bZ 7DeviceStatsMonitor.on_train_batch_end/segment.all.freed"trainer/global_step2:JPAˆ€jØbe BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated"trainer/global_step2:JP¶Y9ogÙbb ?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak"trainer/global_step2:JPÝiÚbd ADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated"trainer/global_step2:JPzsm¸eÛb` =DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak"trainer/global_step2:JP@ŠgAjÜbe BDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak"trainer/global_step2:JP¦#7ÔfÝba >DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed"trainer/global_step2:JPÏ×eÞb` =DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed"trainer/global_step2:JPeäñ4dßb_ DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed"trainer/global_step2:JP }Ädäb_ »£wkèbf CDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed"trainer/global_step2:JPƒê­nébi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed"trainer/global_step2:JPÒÇ¡Zfêba >DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated"trainer/global_step2:JPx×DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current"trainer/global_step2:JP™gRØeób` =DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed"trainer/global_step2:JPlç üiôbd ADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current"trainer/global_step2:JPÏßüpõbk HDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current"trainer/global_step2:JPS„»höbc @DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current"trainer/global_step2:JPÜr÷bm JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated"trainer/global_step2:JPͳP|røbm JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated"trainer/global_step2:JPD’j¹hùbc @DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak"trainer/global_step2:JPx`x–múbh EDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed"trainer/global_step2:JPËå¡bûb] :DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams"trainer/global_step2:JP­Â<‘lübg DDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated"trainer/global_step2:JPR`™býb] :DeviceStatsMonitor.on_train_batch_end/active.all.allocated"trainer/global_step2:JPÜM&gbþb] :DeviceStatsMonitor.on_train_batch_end/allocation.all.freed"trainer/global_step2:JPôÒØ-wÿbr ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JP¸1Ri_€bZ 7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries"trainer/global_step2:JPÈ3ÙVbQ .DeviceStatsMonitor.on_train_batch_end/num_ooms"trainer/global_step2:JP<#àÍe‚b` =DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak"trainer/global_step2:JP¤vHggƒbb ?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed"trainer/global_step2:JP5Œàn„bi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed"trainer/global_step2:JPäv&ýa…b\ 9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak"trainer/global_step2:JP$ëÃl†bg DDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak"trainer/global_step2:JPy¶Õªd‡b_ DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak"trainer/global_step2:JP“Ìldb_ 6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 858293465792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚203182 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚343370 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚888216 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 45829607936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚535174 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 44448616386 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 44472107008 _runtime‚ 24.303781426 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 880047593472 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 44559543808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚147176 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚888216 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚878544 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 883412195328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 P?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 924502661120 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 44472107008 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 trainer/global_step‚49 MDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚350358 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚147431 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 902759121538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_po7•àol.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 880047593472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 147182080 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 44362813150 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚544844 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 A6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚878544Rßhq&•ëL MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚350615 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 _step‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 858293465792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚203182 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚343370 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚888216 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 45829607936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚535174 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 44448616386 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 44472107008 _runtime‚ 24.303781426 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 880047593472 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 44559543808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚147176 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚888216 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚878544 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 883412195328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 P?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 924502661120 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 44472107008 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 trainer/global_step‚49 MDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚350358 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚147431 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 902759121538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 880047593472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 147182080 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 44362813150 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚544844 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 A6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚878544é/żA–b< loss/FRt1/assignment_loss"trainer/global_step2:JPjÚ÷C—b> loss/SRqqt1/assignment_loss"trainer/global_step2:JP®hl A˜b< loss/FBt1/assignment_loss"trainer/global_step2:JPa /‚@™b; loss/FRt2/detection_loss"trainer/global_step2:JPš{›³Bšb= loss/SRqqt2/detection_loss"trainer/global_step2:JPV¨$|@›b; loss/FBt1/detection_loss"trainer/global_step2:JPþ'sŒAœb< loss/FRt2/assignment_loss"trainer/global_step2:JPþÕ<)Ab< loss/FBt2/assignment_loss"trainer/global_step2:JP@nß @žb; loss/FRt1/detection_loss"trainer/global_step2:JPÕÏ¿W-Ÿb( epoch"trainer/global_step2:JPÑOÐHC b> loss/SRqqt2/assignment_loss"trainer/global_step2:JPDz‘ÛB¡b= loss/SRqqt1/detection_loss"trainer/global_step2:JPÔ+¤“@¢b; loss/FBt2/detection_loss"trainer/global_step2:JP²Ó×ã7£b2 loss/total_loss"trainer/global_step2:JPÊÝ„×.¤) _wandbruntime‚24  _runtime‚24%êvºø¥ò _runtime‚ 24.304548004 _step‚3 0loss/SRqqt1/detection_loss‚3.376185128465295 /loss/FBt2/detection_loss‚12.533804817446347 %loss/total_loss‚69.45626686986054 /loss/FRt1/assignment_loss‚5.857690047068768 1loss/SRqqt1/assignment_loss‚4.480335669858115 /loss/FBt1/assignment_loss‚2.325835195454684 /loss/FRt2/detection_loss‚3.5346387326717377 1loss/SRqqt2/detection_loss‚3.8936311226515543 /loss/FBt1/detection_loss‚16.815788571130145 trainer/global_step‚49 /loss/FRt2/assignment_loss‚5.731393517280112 0loss/FBt2/assignment_loss‚2.3012284985904037 .loss/FRt1/detection_loss‚4.087640578308737 epoch‚0 1loss/SRqqt2/assignment_loss‚4.518094990934644 % _timestamp‚1.7858102791147625e+09¡Ãž×ü¦ö trainer/global_step‚49 /loss/FRt2/assignment_loss‚5.731393517280112 0loss/FBt2/assignment_loss‚2.3012284985904037 .loss/FRt1/detection_loss‚4.087640578308737 epoch‚0 1loss/SRqqt2/assignment_loss‚4.518094990934644 % _timestamp‚1.7858102791147625e+09 _runtime‚ 24.304548004 _step‚3 0loss/SRqqt1/detection_loss‚3.376185128465295 /loss/FBt2/detection_loss‚12.533804817446347 %loss/total_loss‚69.45626686986054 /loss/FRt1/assignment_loss‚5.857690047068768 1loss/SRqqt1/assignment_loss‚4.480335669858115 /loss/FBt1/assignment_loss‚2.325835195454684 /loss/FRt2/detection_loss‚3.5346387326717377 1loss/SRqqt2/detection_loss‚3.8936311226515543 /loss/FBt1/detection_loss‚16.815788571130145˜´%6§‚(:’ ìšÅÓج£ƒ gpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚32.0 gpu.0.powerWatts‚103.397' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚34.465666666666664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0§z`ý8¨‚(:­  ìšÅÓÖïêƒ disk.drbd2111.out‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.dm-0.in‚0 disk.dm-0.out‚ 4.0283203125 disk.drbd1938.out‚0) proc.memory.percent‚17.17202663421631 disk.rbd1.out‚ 0.1181640625 disk.drbd1550.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0 cpu‚17.462577472060218 disk.drbd1356.out‚0 disk.drbd1226.in‚0 network.sent‚56542* disk./.usagePercent‚32.844124348176095 disk.drbd1356.in‚0 disk.sda.in‚0 proc.cpu.threads‚29 disk.nvme0n1.in‚0 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2041.in‚0 disk.drbd1673.out‚0" proc.memory.rssMB‚ 11253.859375 disk.drbd2078.out‚0 disk.drbd1289.out‚0( proc.memory.availableMB‚ 38525.453125 disk.drbd1584.in‚0 disk.drbd1552.in‚0 disk.sda2.in‚0 disk.sda.out‚ 1.18359375 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1510.out‚0% memory_percent‚41.214823722839355 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd1552.out‚0 disk.sda2.out‚ 1.18359375 network.recv‚49067 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd2118.out‚0$ disk./.usageGB‚587.2863006591797 disk.drbd2078.in‚0 disk.drbd1817.out‚0! disk.nvme0n1.out‚ 4.0283203125 disk.drbd1931.in‚0 disk.drbd1226.out‚0 disk.rbd0.out‚0ßQ œ©‚(:‘ ûšÅÓÈ—¹ƒ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚34.0 gpu.0.powerWatts‚123.542' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.18066666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0zš91ª‚(:¦  ûšÅÓÃîǃ disk.rbd0.out‚0 disk.drbd2118.in‚0 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0* disk./.usagePercent‚32.844123281486894 disk.drbd1584.in‚0 network.recv‚51599 disk.rbd1.out‚ 0.3603515625 disk.nvme0n1.out‚ 5.197265625 disk.drbd1356.out‚0 disk.drbd2078.in‚0$ memory_percent‚41.21448993682861 network.sent‚101381 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1817.out‚0 disk.drbd1938.out‚0 disk.drbd1817.in‚0 disk.drbd1931.out‚0 disk.drbd2118.out‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.rbd0.in‚0 disk.drbd1938.in‚0 disk.drbd1356.in‚0 disk.drbd1226.in‚0 cpu‚16.92882688105688 disk.sda.out‚1.484375 disk.drbd2078.out‚0( proc.memory.availableMB‚ 38525.671875$ disk./.usageGB‚587.2862815856934 disk.dm-0.in‚0 disk.sda1.in‚0 disk.sda2.out‚1.484375 disk.drbd1289.in‚0 disk.drbd1552.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0" proc.memory.rssMB‚ 11253.859375 proc.cpu.threads‚29 disk.drbd1552.out‚0 disk.drbd1226.out‚0 disk.sda2.in‚0 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.dm-0.out‚ 5.197265625) proc.memory.percent‚17.17202663421631 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.drbd1931.in‚0 disk.sda.in‚0és±D.«) _wandbruntime‚45  _runtime‚45ÈÞÖì½ _runtime‚ 45.300096636 _step‚4 ' lr-AdamW/pg1‚2.5514464620453036e-06 ' lr-AdamW/pg2‚2.5514464620453036e-06 trainer/global_step‚99 % _timestamp‚1.7858103001101923e+09?7ÙÇ­Á _step‚4 ' lr-AdamW/pg1‚2.5514464620453036e-06 ' lr-AdamW/pg2‚2.5514464620453036e-06 trainer/global_step‚99 % _timestamp‚1.7858103001101923e+09 _runtime‚ 45.300096636Vðì.®) _wandbruntime‚45  _runtime‚45þwOŽ|'¯öN QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 87373803436 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚99 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 87588622848 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 1721218980864 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 90251077120 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 87676052480 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1731188 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚ 45.301746323 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚683141 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚396095 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚512 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 87676052480 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 1808895033344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 1726536014848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1721530 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚522 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 1726560918528 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1062309 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 87459606560 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚668877 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 1678561909472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 1808895033344 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1052653 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 1721201941504 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚396097 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 1721201941504 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 1808790564352 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 1721218980864 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚287046 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 90116208640 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 1816652223488 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 1678578948832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 1808790564352 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _step‚5"Âϯ°úN K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1052653 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 1765935712908 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7858103001110914e+09 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1062309 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚668879 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚668879 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 1726536014848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1721530 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚522 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 1726560918528 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1062309 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 87459606560 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚668877 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 1678561909472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 1808895033344 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1052653 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocat³•coed‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 1721201941504 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚396097 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 1721201941504 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 1808790564352 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 1721218980864 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚287046 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 90116208640 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 1816652223488 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 1678578948832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 1808790564352 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _step‚5 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 87373803436 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚99 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 87588622848 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 1721218980864 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 90251077120 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 87676052480 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1731188 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚ 45.301746323 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚683141 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚396095 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚512 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 87676052480 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 1808895033344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 H6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1063176 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 1826941423104 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 1783656201046 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚243 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚290166 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚512 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1072846 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚ 45.656322098 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 88468242432 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 1826941423104 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 1826836946944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚675520 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 91018568192 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 1834761699840 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 91151332352 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 88337079894 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 1743743131648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 1738385743872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 88555679232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 1738368704512 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 1738368704512 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚289925 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 1695319121152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 1743768035328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 88251276658 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚99 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚690200 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858103004660525e+09 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚675522 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1072846 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚675520 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚689957 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚400034 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 1695302081792 _step‚6 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1748368 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 88468242432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1748368 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 1743768035328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 88251276658 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚99 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚690200 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858103004660525e+09 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚675522 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1072846 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚675520 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚689957 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚400034 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 1695302081792 _step‚6 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1748368 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 88468242432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1748368 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1063176 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 1826941423104 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 1783656201046 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚243 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚290166 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚512 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1072846 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚ 45.656322098 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 88468242432 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 1826941423104 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 1826836946944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚675520 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 91018568192 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 1834761699840 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 91151332352 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 88337079894 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 1743743131648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 1738385743872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 88555679232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 1738368704512 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 1738368704512 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚289925 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 1695319121152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 2711124850176 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 2711124850176 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1001027 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1580655 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 2579540091904 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1590311 L@DeviceStatsMonitor.on_train_batch_start/inactive_splitV8`h.all.freed‚1023067 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚592947 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 2579540091904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚265 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 2722230393344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 2646832792416 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 131348070068 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 131262266944 _runtime‚ 66.526647606 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 135338538496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚2581682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1001027 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 2579557131264 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 trainer/global_step‚149 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 2515587564832 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2591340 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 2722388068352 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 2586891854848 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858103213332083e+09 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚430122 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚522 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 2711229319168 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 2646935634900 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 131584758272 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1023332 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚263 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 135471309824 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 2515570525472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1001027 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 2579557131264 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 trainer/global_step‚149 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 2515587564832 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2591340 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 2722388068352 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 2586891854848 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858103213332083e+09 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚430122 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚522 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 2711229319168 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 2646935634900 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 131584758272 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1023332 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚263 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 135471309824 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 2515570525472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 2711124850176 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 2711124850176 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1001027 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1580655 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 2579540091904 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1590311 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1023067 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚592947 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 2579540091904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚265 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 2722230393344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 2646832792416 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 131348070068 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 131262266944 _runtime‚ 66.526647606 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 135338538496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚2581682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1007672 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 2596706854912 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 132551814656 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚522 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 132464377856 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 2740495768064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 2664553280554 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 2664450437958 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2598848 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 2596723894272 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1007670 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1007670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚10 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1029887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 2729171232768 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 2532327737152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 2532310697792 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1591178 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚596884 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2608520 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 2729275708928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2608520 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1600848 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚433261 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚512 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚2598848 trainer/global_step‚149 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 132225543402 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 136245420032 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 2604123875328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1007672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0ß]p"Á”M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 132225543402 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 136245420032 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 2604123875328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1007672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1007672 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 2596706854912 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 132551814656 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚522 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 132464377856 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 2740495768064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 2664553280554 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 2664450437958 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2598848 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 2596723894272 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1007670 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1007670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚10 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1029887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 2729171232768 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 2532327737152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 2532310697792 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1591178 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚596884 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2608520 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 2729275708928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2608520 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1600848 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚433261 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚512 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚2598848 trainer/global_step‚149 )gZ.Â) _wandbruntime‚66  _runtime‚66}§áøÃò /loss/FRt1/detection_loss‚3.6690019963497527 /loss/FRt2/detection_loss‚3.6544869755921154 /loss/FBt1/detection_loss‚15.271569799631834 /loss/FRt1/assignment_loss‚5.733889001139095 /loss/FRt2/assignment_loss‚5.720224323480026 0loss/SRqqt2/detection_loss‚3.486379630377759 /loss/FBt2/detection_loss‚15.767359500345977 trainer/global_step‚149 _step‚11 1loss/SRqqt1/assignment_loss‚4.533540909269215 .loss/FBt1/assignment_loss‚2.33242329955101 /loss/FBt2/assignment_loss‚2.272729324257892 epoch‚0 _runtime‚ 66.887251642 2loss/SRqqt2/assignment_loss‚4.5869258383046025 /loss/SRqqt1/detection_loss‚3.19117414397476 %loss/total_loss‚70.21970474227405 % _timestamp‚1.7858103216974638e+09È ìßüÄö 2loss/SRqqt2/assignment_loss‚4.5869258383046025 /loss/SRqqt1/detection_loss‚3.19117414397476 %loss/total_loss‚70.21970474227405 % _timestamp‚1.7858103216974638e+09 /loss/FRt1/detection_loss‚3.6690019963497527 /loss/FRt2/detection_loss‚3.6544869755921154 /loss/FBt1/detection_loss‚15.271569799631834 /loss/FRt1/assignment_loss‚5.733889001139095 /loss/FRt2/assignment_loss‚5.720224323480026 0loss/SRqqt2/detection_loss‚3.486379630377759 /loss/FBt2/detection_loss‚15.767359500345977 trainer/global_step‚149 _step‚11 1loss/SRqqt1/assignment_loss‚4.533540909269215 .loss/FBt1/assignment_loss‚2.33242329955101 /loss/FBt2/assignment_loss‚2.272729324257892 epoch‚0 _runtime‚ 66.887251642 2ЄÂÅ‚(:’ ™›ÅÓÓó€ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚36.0 gpu.0.powerWatts‚132.633' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.211000000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ºøGÚ7Æ‚(:¬  ™›ÅÓÉÌƃ disk.drbd1828.out‚0 disk.drbd1584.in‚0 disk.drbd1965.in‚0) proc.memory.availableMB‚ 38524.8984375% memory_percent‚41.215670108795166 network.recv‚56111 disk.drbd1510.in‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 cpu‚16.213647504433975 network.sent‚149479 disk.drbd1817.out‚0 disk.drbd1938.out‚0 disk.drbd1931.out‚0 disk.rbd0.out‚0 disk.drbd2111.out‚0* disk./.usagePercent‚32.858240273091546 disk.drbd1584.out‚0 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.sda.in‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.sda2.out‚3.140625 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.rbd1.out‚ 0.392578125" proc.memory.rssMB‚ 11253.859375$ disk./.usageGB‚587.5387077331543 disk.drbd1356.out‚0 disk.sda.out‚3.140625 disk.drbd1289.out‚0 disk.sda1.in‚0 disk.drbd1931.in‚0 proc.cpu.threads‚29 disk.drbd1550.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0 disk.drbd2041.in‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.dm-0.out‚ 58.1357421875 disk.drbd2118.out‚0" disk.nvme0n1.out‚ 58.1357421875 disk.drbd1965.out‚0) proc.memory.percent‚17.17202663421631 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1552.out‚0 disk.sda2.in‚0 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.drbd1828.in‚0 disk.dm-0.in‚0 disk.drbd1817.in‚0Ý‘ˆ.Ç) _wandbruntime‚87  _runtime‚87Ïî ÍÃȽ & lr-AdamW/pg1‚5.128665110575913e-06 & lr-AdamW/pg2‚5.128665110575913e-06 trainer/global_step‚199 % _timestamp‚1.7858103427411876e+09 _runtime‚ 87.931227307 _step‚12ÏšÖÇÉÁ _runtime‚ 87.931227307 _step‚12 & lr-AdamW/pg1‚5.128665110575913e-06 & lr-AdamW/pg2‚5.128665110575913e-06 trainer/global_step‚199 % _timestamp‚1.7858103427411876e+09 ÅÅ*Ô.Ê) _wandbruntime‚87  _runtime‚87#ñŒ'ˆO UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1333179 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1333177 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 175150730452 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 175668323328 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2118313 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 3352579141472 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚573322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚516 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 3447272598528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 3613563604992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚789795 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 3527729871924 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 3352596180832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 3437895281664 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 3613563604992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 3437895281664 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 175236533576 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 175580893696 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3441834 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2118313 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 3613459136000 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 175580893696 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 3447247694848 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 180617964544 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚13 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 175668323328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚ 87.935635607 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1333177 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 3437878242304 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3451492 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚789797 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 180483096064 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 3613459136000 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 3627890563072 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2108657 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992²[²s'ÌŠO L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1363117 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚789795 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 3527729871924 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 3352596180832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 3437895281664 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 3613563604992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 3437895281664 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 175236533576 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 175580893696 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3441834 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2118313 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 3613459136000 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 175580893696 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 3447247694848 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 180617964544 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚13 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 175668323328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚ 87.935635607 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1333177 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 3437878242304 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3451492 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚789797 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 180483096064 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 3613459136000 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 3627890563072 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2108657 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1333179 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1333177 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 175150730452 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 175668323328 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2118313 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 3352579141472 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚573322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚516 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 3447272598528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 3613563604992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1370131 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚199 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3459000 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚243 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 181515510272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 3455045005312 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 176460513280 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 F6DeviceStatsMonitor.oDoº–n_train_batch_end/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1339820 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2128850 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1339822 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 3545347517466 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 3369336353152 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1339822 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚14 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚793732 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 3631609994752 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 176460513280 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 3464454811648 NDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2128850 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚576397 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1369888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 176114006910 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858103431010895e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 3645995225600 _runtime‚ 88.291424007 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3468672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3468672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2119180 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 3455045005312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1339820 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 176547950080 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚576156 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 3464479715328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 181391134720 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 176547950080 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 3645845946368 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712õ”¶{š&Ï”M MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1339820 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2128850 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1339822 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 3545347517466 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 3369336353152 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1339822 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚14 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚793732 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 3631609994752 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 176460513280 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 3464454811648 NDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2128850 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚576397 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1369888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 176114006910 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858103431010895e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 3645995225600 _runtime‚ 88.291424007 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3468672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3468672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2119180 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 3455045005312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1339820 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 176547950080 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚576156 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 3464479715328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 181391134720 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 176547950080 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 3645845946368 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 3631609994752 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2119180 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1370131 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚199 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3459000 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚243 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 181515510272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 3455045005312 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 176460513280 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 Ø .Ð) _wandbruntime‚88  _runtime‚88Ý·gûÑõ 0loss/SRqqt2/detection_loss‚4.312225780089697 /loss/FBt1/detection_loss‚13.999127454482592 /loss/FRt1/assignment_loss‚5.750562618518698 1loss/SRqqt1/assignment_loss‚4.505067574892113 1loss/SRqqt2/assignment_loss‚4.573231236139933 1loss/SRqqt1/detection_loss‚2.4737070213547714 % _timestamp‚1.7858103431024337e+09 /loss/FRt1/detection_loss‚3.8794473570653762 trainer/global_step‚199 epoch‚0 /loss/FRt2/assignment_loss‚5.824674753495205 0loss/FBt1/assignment_loss‚2.2791942908213687 0loss/FBt2/assignment_loss‚2.2593667030334474 .loss/FRt2/detection_loss‚4.128464106424355 _runtime‚ 88.292263699 /loss/FBt2/detection_loss‚14.429792563915253 %loss/total_loss‚68.41486146023281 _step‚15(€«ÿÒù 0loss/FBt2/assignment_loss‚2.2593667030334474 .loss/FRt2/detection_loss‚4.128464106424355 _runtime‚ 88.292263699 /loss/FBt2/detection_loss‚14.429792563915253 %loss/total_loss‚68.41486146023281 _step‚15 0loss/SRqqt2/detection_loss‚4.312225780089697 /loss/FBt1/detection_loss‚13.999127454482592 /loss/FRt1/assignment_loss‚5.750562618518698 1loss/SRqqt1/assignment_loss‚4.505067574892113 1loss/SRqqt2/assignment_loss‚4.573231236139933 1loss/SRqqt1/detection_loss‚2.4737070213547714 % _timestamp‚1.7858103431024337e+09 /loss/FRt1/detection_loss‚3.8794473570653762 trainer/global_step‚199 epoch‚0 /loss/FRt2/assignment_loss‚5.824674753495205 0loss/FBt1/assignment_loss‚2.2791942908213687Á L‘;Ó‚(:°  ¨›ÅÓµº¹ƒ disk.sda2.out‚ 3.51953125 disk.rbd1.out‚ 0.392578125 disk.nvme0n1.in‚0 disk.sda.in‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1552.out‚0 proc.cpu.threads‚29" proc.memory.rssMB‚ 11253.859375 disk.drbd1550.out‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1965.out‚0 cpu‚15.960803324609854 disk.dm-0.out‚264.5693359375 disk.drbd1510.in‚0 disk.sda1.in‚0# disk./.usageGB‚587.538745880127 disk.drbd1356.in‚0 disk.drbd1931.in‚0 disk.drbd2111.out‚0) proc.memory.percent‚17.17202663421631) disk./.usagePercent‚32.85824240646996 disk.drbd1938.in‚0 disk.drbd1226.in‚0 network.sent‚194363 disk.drbd2041.out‚0 disk.rbd0.out‚0* proc.memory.availableMB‚38525.03515625 disk.drbd1550.in‚0 disk.drbd1289.in‚0 network.recv‚58867 disk.sda2.in‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd2078.out‚0 disk.drbd1938.out‚0$ memory_percent‚41.21546149253845# disk.nvme0n1.out‚264.5693359375 disk.dm-0.in‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd2041.in‚0 disk.sda.out‚ 3.51953125 disk.drbd2078.in‚0 disk.drbd1584.out‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd1356.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0& ×Ô‚(:… ¨›ÅÓæ껃 gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚36.0 gpu.0.powerWatts‚97.035' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.345 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 _¡œÕ‚(:‘ ·›ÅÓ½Û…ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚36.0 gpu.0.powerWatts‚113.224' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.74133333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ñ hñ:Ö‚(:¯  ·›ÅÓ»·Ñƒ disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd1226.in‚0 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.drbd1938.in‚0" proc.memory.rssMB‚ 11253.859375 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd2078.out‚0* disk./.usagePercent‚32.858240273091546# disk.nvme0n1.out‚264.8310546875 disk.drbd1356.in‚0 disk.drbd1356.out‚0$ disk./.usageGB‚587.5387077331543 disk.dm-0.out‚264.8310546875 disk.drbd1552.in‚0 disk.drbd1584.in‚0 network.sent‚239382 disk.sda1.out‚0 disk.rbd1.out‚ 0.548828125 disk.drbd1285.out‚0 disk.drbd1817.out‚0 disk.drbd1931.in‚0 disk.drbd1965.in‚0 disk.dm-0.in‚0 disk.drbd2118.in‚0 disk.drbd1226.out‚0 disk.drbd1673.in‚0 disk.rbd0.in‚0 disk.drbd1584.out‚0 network.recv‚61468 disk.drbd1510.out‚0 disk.nvme0n1.in‚0 disk.sda.in‚0" memory_percent‚41.215980052948 disk.drbd1289.in‚0 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.drbd1289.out‚0 disk.drbd2118.out‚0 disk.sda2.in‚0 disk.sda2.out‚ 4.08203125) proc.memory.availableMB‚ 38524.6953125 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.drbd1938.out‚0 disk.drbd1965.out‚0) proc.memory.percent‚17.17202663421631 cpu‚15.754520608130644 disk.drbd1828.in‚0 disk.drbd1817.in‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 proc.cpu.threads‚29 disk.drbd1285.in‚0 disk.drbd1673.out‚0 disk.sda.out‚ 4.082031251»{0×+ _wandbruntime‚109  _runtime‚109ªŠ [Äؾ & lr-AdamW/pg1‚6.417274434841219e-06 & lr-AdamW/pg2‚6.417274434841219e-06 trainer/global_step‚249 % _timestamp‚1.7858103642972999e+09 _runtime‚ 109.487238144 _step‚16qV¯ÈÙ & lr-AdamW/pg1‚6.417274434841219e-06 & lr-AdamW/pg2‚6.417274434841219e-06 trainer/global_step‚249 % _timestamp‚1.7858103642972999e+09 _runtime‚ 109.487238144 _step‚16 F}‘0Ú+ _wandbruntime‚109  _runtime‚109ÁQ•Œ'Û†O =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 219039193960 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚242 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 4515897890816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 4533235820032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 4296233432064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 4533391397888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 4189587757472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7858103642982569e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 4408729793916 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 4408626951432 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 219664458752 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1665327 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _runtime‚ 109.49166909 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2646315 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚716863 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚517 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 4307628438528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 4515793421824 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 4296233432064 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 225632285184 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2636659 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2636659 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚240 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚716623 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 4515793421824 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 225762959360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 219577029120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚986647 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 4307603534848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1665327 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130Všñ}&ÜŠO OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1665329 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 4515897890816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/requ¯lþpj&ested_bytes.all.allocated‚ 4408729793916 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 4408626951432 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 219664458752 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1665327 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _runtime‚ 109.49166909 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2646315 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚716863 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚517 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 4307628438528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 4515793421824 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 4296233432064 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 225632285184 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2636659 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2636659 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚240 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚716623 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 4515793421824 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 225762959360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 219577029120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚986647 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 4307603534848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1665327 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 219039193960 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚242 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 4515897890816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 4533235820032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 4296233432064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 4533391397888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 4189587757472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7858103642982569e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1671970 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 N6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 4313383155712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 4206327929792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 219916667182 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2656852 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 220456648704 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚527 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚719465 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 226665532928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1671970 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1671972 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1671972 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _step‚18 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2656852 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 226530671616 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 4426244596974 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 4313383155712 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 4324835555328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4328824 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚990584 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1710047 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 4313400195072 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 4533944280576 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚ 109.853821794 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4319152 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130…T›&ß•M J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4328824 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚990584 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1710047 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 4313400195072 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 4533944280576 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚ 109.853821794 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4319152 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4328824 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 4324810651648 % _timestamp‚1.7858103646634803e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 4551501088256 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2647182 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 4533839804416 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 220456648704 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1671970 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 N6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 4313383155712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 4206327929792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 219916667182 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2656852 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 220456648704 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚527 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚719465 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 226665532928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1671970 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1671972 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1671972 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _step‚18 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2656852 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 226530671616 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 4426244596974 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 4313383155712 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 4324835555328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251TüËÝ0à+ _wandbruntime‚109  _runtime‚109ZLEäûáõ %loss/total_loss‚80.93622112674174 epoch‚0 /loss/FRt2/assignment_loss‚5.640995456747813 2loss/SRqqt1/assignment_loss‚4.5555943981293705 1loss/SRqqt2/assignment_loss‚4.548645936049424 0loss/FBt1/assignment_loss‚2.2296740412712097 0loss/FBt2/assignment_loss‚2.2971394658088684 0loss/SRqqt2/detection_loss‚4.114177222375746 .loss/FRt1/detection_loss‚3.992135109671627 1loss/SRqqt1/detection_loss‚2.7948130587896993 % _timestamp‚1.7858103646647282e+09 /loss/FBt1/detection_loss‚17.977123391628265 /loss/FBt2/detection_loss‚22.337183605879545 _runtime‚ 109.854556003 _step‚19 .loss/FRt1/assignment_loss‚5.82946688295847 .loss/FRt2/detection_loss‚4.619272557431704 trainer/global_step‚249_‡½Ãÿâù /loss/FBt1/detection_loss‚17.977123391628265 /loss/FBt2/detection_loss‚22.337183605879545 _runtime‚ 109.854556003 _step‚19 .loss/FRt1/assignment_loss‚5.82946688295847 .loss/FRt2/detection_loss‚4.619272557431704 trainer/global_step‚249 %loss/total_loss‚80.93622112674174 epoch‚0 /loss/FRt2/assignment_loss‚5.640995456747813 2loss/SRqqt1/assignment_loss‚4.5555943981293705 1loss/SRqqt2/assignment_loss‚4.548645936049424 0loss/FBt1/assignment_loss‚2.2296740412712097 0loss/FBt2/assignment_loss‚2.2971394658088684 0loss/SRqqt2/detection_loss‚4.114177222375746 .loss/FRt1/detection_loss‚3.992135109671627 1loss/SRqqt1/detection_loss‚2.7948130587896993 % _timestamp‚1.7858103646647282e+09lER ‘ã‚(:† Æ›ÅÓÝꙃ gpu.0.gpu‚22.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚37.0 gpu.0.powerWatts‚120.873' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚40.291 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ýãç:ä‚(:²  Æ›ÅÓºûûƒ network.recv‚62794 disk.sda.in‚0 disk.sda2.out‚ 4.30078125# disk.nvme0n1.out‚265.0810546875 disk.drbd1584.out‚0 disk.drbd2078.in‚0 disk.dm-0.out‚265.0810546875* disk./.usagePercent‚32.858240273091546 disk.drbd1828.out‚0 disk.sda1.out‚0% memory_percent‚41.217029094696045 disk.drbd1226.out‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.drbd2041.in‚0 disk.drbd1285.out‚0 disk.drbd1289.out‚0" proc.memory.rssMB‚ 11253.859375 disk.drbd1356.out‚0 disk.rbd0.out‚0 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.drbd1550.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.sda.out‚ 4.30078125 disk.drbd1965.out‚0 disk.drbd1552.in‚0 disk.rbd1.out‚ 0.548828125 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.drbd1285.in‚0) proc.memory.percent‚17.17202663421631 disk.drbd1817ìÿûE.out‚0 disk.dm-0.in‚0 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.drbd1938.out‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0 disk.drbd1828.in‚0) proc.memory.availableMB‚ 38524.0078125 disk.drbd1938.in‚0 disk.sda2.in‚0 disk.drbd2041.out‚0 disk.drbd1550.in‚0 disk.drbd2078.out‚0$ disk./.usageGB‚587.5387077331543 cpu‚15.576555635010969 proc.cpu.threads‚29 network.sent‚242675 disk.nvme0n1.in‚0 disk.drbd1584.in‚0 disk.drbd1510.out‚0ZWÙI0å+ _wandbruntime‚131  _runtime‚131ŠäWöÄæ¾ trainer/global_step‚299 % _timestamp‚1.7858103864142559e+09 _runtime‚ 131.604219596 _step‚20 & lr-AdamW/pg1‚7.705883759106523e-06 & lr-AdamW/pg2‚7.705883759106523e-06+K„Èç _step‚20 & lr-AdamW/pg1‚7.705883759106523e-06 & lr-AdamW/pg2‚7.705883759106523e-06 trainer/global_step‚299 % _timestamp‚1.7858103864142559e+09 _runtime‚ 131.604219596@ £0è+ _wandbruntime‚131  _runtime‚131ñoÂ'é‰O GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2044019 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 5154554543104 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 262927657468 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 5418127707648 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 5154554543104 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3174317 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚21 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1997477 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 5167959374848 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 263013460592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚5162138 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 5154571582464 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚860524 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 270930186240 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 263660594176 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5171796 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 5026596373472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 5289524030940 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1997477 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7858103864152803e+09 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3164661 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚299 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 263573164544 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 5167984278528 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 5289626873424 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3174317 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 5418127707648 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5026613412832 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 5418232176640 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2044245 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚224 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1997479 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 5438914464768 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1183497 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320rm“'êO ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 5026596373472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 5289524030940 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1997477 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7858103864152803e+09 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3164661 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚299 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 263573164544 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 5167984278528 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 5289626873424 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3174317 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 5418127707648 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5026613412832 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 5418232176640 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2044245 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚224 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1997479 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 5438914464768 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1183497 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2044019 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 5154554543104 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 262927657468 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 5418127707648 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 5154554543104 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3174317 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚21 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1997477 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 5167959374848 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 263013460592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚5162138 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 5154571582464 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚860524 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 270930186240 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 263660594176 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5171796 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0ï=”0ë+ _wandbruntime‚131  _runtime‚131¯f;˜&ì’M MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2004120 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 5457035959808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858103867838945e+09 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 5307244519078 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3175184 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚ 131.97420793 NDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2004122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1187432 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚863611 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 5185191395328 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚5188976 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 5171721306112 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3175184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 5043336545792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _step‚22 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 5436278566400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 263805130690 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 5171738345472 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 5436174090240 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 5307141676482 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3184854 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 5043353585152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2004120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5188976 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3184854 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2050790 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5179304 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 5185166491648 trainer/global_step‚299 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 5171738345472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚863358 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 263890933926 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1187434 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 271720188928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2004122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1187432 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚863611 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 5185191395328 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚5188976 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 5171721306112 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3175184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 5043336545792 SDDeviceStatsMonitor.on_train_batch_end/reserved_byJ—þ» tes.large_pool.peak‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _step‚22 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 5436278566400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 263805130690 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 5171738345472 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 5436174090240 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 5307141676482 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3184854 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 5043353585152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2004120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5188976 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3184854 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2050790 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5179304 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 5185166491648 trainer/global_step‚299 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 5171738345472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚863358 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 263890933926 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1187434 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 271720188928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚ 131.97420793 NDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 6012892693504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3692663 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1004583 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 6012909732864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 307656729600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 306901924100 _step‚25 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2384678 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1380347 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 316120520704 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5863622028832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3702319 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 6320461993472 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2329627 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 315981457920 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 6170523952932 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 5863604989472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640‡M’úŽO WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 6320566462464 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 307569299968 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 6012892693504 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3702319 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 6012909732864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 307656729600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 306901924100 _step‚25 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2384678 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1380347 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 316120520704 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5863622028832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3702319 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 6320461993472 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2329627 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 315981457920 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 6170523952932 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 5863604989472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2384930 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 6012892693504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3692663 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1004583 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 308536356352 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6049128 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 6188141598586 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 308448919552 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 6188038755990 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2391465 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2336272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858104081441562e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 N8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 317020399104 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3703186 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 308448919552 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 6338612852224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚349 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 6362405772288 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2336270 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 _runtime‚ 153.334509094 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1384284 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1007183 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 316883440640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3703186 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 6030059456512 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3712856 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3712856 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 6362567634432 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 6030059456512 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 307779397434 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0I)1ÕŸ&ý™M >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 317020399104 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3703186 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 308448919552 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 6338612852224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚349 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 6362405772288 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2336270 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 _runtime‚ 153.334509094 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1384284 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1007183 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 316883440640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3703186 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 6030059456512 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3712856 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3712856 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 6362567634432 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 6030059456512 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 307779397434 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 308536356352 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 308536356352 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6049128 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 6188141598586 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 308448919552 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 6188038755990 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2391465 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2336272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858104081441562e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 N8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 trainer/global_step‚399 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 7222796279296 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 351565435392 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2725603 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 350790387608 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 350704584484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 7051318189956 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 6700613605472 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 7222900748288 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 7222796279296 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 7249990581248 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 361161851392 % _timestamp‚1.7858104298348045e+09 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 7222900748288 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4220665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1577197 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚6892100 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2725860 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 6871247883264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 6700630644832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚29 O@DeviceStatsMonitor.on_train_batch_sta[›‘,0 rt/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 6871230843904 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1148663 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 7249832906240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2661777 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 6888671054848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 6871247883264 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2661779 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 6888695958528 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4220665 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 6871230843904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 351565435392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024OÄ{S•'ˆO OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4230321 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1148408 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 361294622720 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4230321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2725603 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 350790387608 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 350704584484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 7051318189956 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 6700613605472 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 7222900748288 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 7222796279296 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 7249990581248 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 361161851392 % _timestamp‚1.7858104298348045e+09 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 7222900748288 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4220665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1577197 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚6892100 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2725860 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 6871247883264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 6700630644832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚29 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 6871230843904 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1148663 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 7249832906240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2661777 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 6888671054848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 6871247883264 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2661779 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 6888695958528 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4220665 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 6871230843904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 351565435392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 trainer/global_step‚399 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 7222796279296 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 351565435392 H6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2668422 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4240858 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 6717353777792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚30 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6909280 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2732643 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1581134 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 7268098485760 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6899608 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚261 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 6905878171648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4240858 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2732382 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 7240947138048 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 7240947138048 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858104301945808e+09 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 352445054976 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6909280 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 362062646272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 352532491776 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 trainer/global_step‚399 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 6888397606912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 _runtime‚ 175.384902847 NDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4231188 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 352532491776 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640e•ŠM$‹™M MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2668420 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 7068935835498 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4231188 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 352532491776 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2668422 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4240858 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 6717353777792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚30 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6909280 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2732643 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1581134 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 7268098485760 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6899608 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚261 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 6905878171648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4240858 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2732382 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 7240947138048 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 7240947138048 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858104301945808e+09 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 352445054976 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6909280 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 362062646272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 352532491776 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 trainer/global_step‚399 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 6888397606912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 _runtime‚ 175.384902847 Nû Æ”À ' lr-AdamW/pg1‚1.1571711731902438e-05 ' lr-AdamW/pg2‚1.1571711731902438e-05 trainer/global_step‚449 % _timestamp‚1.7858104514630437e+09 _runtime‚ 196.652995061 _step‚32#ôŠÊ•Ä ' lr-AdamW/pg1‚1.1571711731902438e-05 ' lr-AdamW/pg2‚1.1571711731902438e-05 trainer/global_step‚449 % _timestamp‚1.7858104514630437e+09 _runtime‚ 196.652995061 _step‚32 QØÃ0–+ _wandbruntime‚196  _runtime‚196þûO‘'—‹O ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 395649000448 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4758323 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 8125130565120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 7729586033664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 7749051798528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 7537639260832 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2993927 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1774045 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 7537622221472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 7729586033664 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 8155389281280 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚7752252 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 7729568994304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 7729568994304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚ 196.654792195 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2993929 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 406493060608 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 8125130565120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4748667 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 8125235034112 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3065401 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 406362386432 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚33 % _timestamp‚1.7858104514640563e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4758323 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 395561570816 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2993927 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 394593047992 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0Læ •'˜O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 7537639260832 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2993927 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1774045 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 7537622221472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 7729586033664 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 8155389281280 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚7752252 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 7729568994304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 7729568994304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚ 196.654792195 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2993929 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 406493060608 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 8125130565120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4748667 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 8125235034112 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3065401 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 406362386432 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚33 % _timestamp‚1.7858104514640563e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4758323 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 395561570816 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2993927 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 394593047992 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 395649000448 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4758323 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 8125130565120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 7729586033664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 7749051798528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136!O ‹0™+ _wandbruntime‚197  _runtime‚197䟰))š•M KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3072435 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚ 197.016966172 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 8143281423872 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 395556324450 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 396441190400 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 8143176947712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 7746735757312 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3000570 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 7554362393792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 7746752796672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚449 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 7554379433152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858104518266594e+09 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3072184 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1777982 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 7766234011648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3000570 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 396441190400 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1777984 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1294202 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 395470521214 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 396528627200 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 7949832915006 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚7769432 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.hÝ°³rallocated‚ 407398919168 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4759190 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 NDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7769432 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4768860 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4759190 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 7746735757312 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1294451 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 8173500166656 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 407266155008 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚7759760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚34 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3000572 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4768860 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 396528627200 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 7746752796672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚449 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 7554379433152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858104518266594e+09 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3072184 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1777982 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 7766234011648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3000570 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 396441190400 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1777984 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1294202 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 395470521214 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 396528627200 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 7949832915006 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚7769432 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 407398919168 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4759190 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 NDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7769432 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4768860 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4759190 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 7746735757312 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1294451 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 8173500166656 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 407266155008 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚7759760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚34 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3000572 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4768860 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 396528627200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 395556324450 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032"ÿ)þœ0œ+ _wandbruntime‚197  _runtime‚197éZKûõ 1loss/SRqqt1/detection_loss‚1.9640640084030507 0loss/SRqqt2/detection_loss‚6.034797294522232 /loss/FBt2/detection_loss‚11.564151790057277 $loss/total_loss‚84.2926479545263 _step‚35 /loss/FRt1/assignment_loss‚5.591533732414246 0loss/SRqqt2/assignment_loss‚4.52394126496225 .loss/FRt1/detection_loss‚4.105998128652573 epoch‚0 2loss/SRqqt1/assignment_loss‚4.5352839518196975 0loss/FBt2/assignment_loss‚1.8406581426488942 /loss/FRt2/detection_loss‚3.7491000470789997 trainer/global_step‚449 % _timestamp‚1.7858104518278658e+09 _runtime‚ 197.017646938 /loss/FRt2/assignment_loss‚5.878982370549982 0loss/FBt1/assignment_loss‚1.8310800899158826 /loss/FBt1/detection_loss‚32.673057133501224ºU\ÿžù 0loss/FBt2/assignment_loss‚1.8406581426488942 /loss/FRt2/detection_loss‚3.7491000470789997 trainer/global_step‚449 % _timestamp‚1.7858104518278658e+09 _runtime‚ 197.017646938 /loss/FRt2/assignment_loss‚5.878982370549982 0loss/FBt1/assignment_loss‚1.8310800899158826 /loss/FBt1/detection_loss‚32.673057133501224 1loss/SRqqt1/detection_loss‚1.9640640084030507 0loss/SRqqt2/detection_loss‚6.034797294522232 /loss/FBt2/detection_loss‚11.564151790057277 $loss/total_loss‚84.2926479545263 _step‚35 /loss/FRt1/assignment_loss‚5.591533732414246 0loss/SRqqt2/assignment_loss‚4.52394126496225 .loss/FRt1/detection_loss‚4.105998128652573 epoch‚0 2loss/SRqqt1/assignment_loss‚4.5352839518196975#›ô3œŸ‚(:‘  œÅÓü׃ gpu.0.gpu‚31.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚94.491' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚31.496999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0;‡i < ‚(:±   œÅÓÀðäƒ disk.sda.in‚0 network.recv‚76130 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.drbd1510.in‚0 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.dm-0.in‚0) proc.memory.availableMB‚ 38523.5078125* proc.memory.percent‚17.172133922576904 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.sda2.in‚0$ disk./.usageGB‚587.5379753112793 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.sda2.out‚ 6.8515625 disk.drbd1584.in‚0 disk.drbd1965.out‚0 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd1673.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.rbd0.in‚0 disk.drbd1552.in‚0 disk.drbd1938.in‚0 disk.rbd1.out‚ 0.76953125 disk.nvme0n1.in‚0 disk.sda.out‚ 6.8515625 disk.drbd1226.in‚0 disk.drbd1673.out‚0 disk.drbd1931.in‚0 network.sent‚429411 disk.drbd1938.out‚0# proc.memory.rssMB‚ 11253.9296875 proc.cpu.threads‚29) disk./.usagePercent‚32.85819931222611 disk.drbd1289.out‚0 disk.drbd1356.in‚0 disk.drbd1584.out‚0$ memory_percent‚41.21779203414917 disk.drbd2111.in‚0 disk.drbd2078.in‚0 disk.drbd1817.in‚0 cpu‚14.936102912337246! disk.dm-0.out‚266.78955078125 disk.sda1.in‚0 disk.drbd1552.out‚0$ disk.nvme0n1.out‚266.78955078125 disk.drbd2041.in‚0…N,ê0¡+ _wandbruntime‚218  _runtime‚218Îû,…Æ¢À ' lr-AdamW/pg2‚1.2860321056167744e-05 trainer/global_step‚499 % _timestamp‚1.7858104730104396e+09 _runtime‚ 218.200393177 _step‚36 ' lr-AdamW/pg1‚1.2860321056167744e-05àÛÃ检 _runtime‚ 218.200393177 _step‚36 ' lr-AdamW/pg1‚1.2860321056167744e-05 ' lr-AdamW/pg2‚1.2860321056167744e-05 trainer/global_step‚499 % _timestamp‚1.7858104730104396e+09$5nÀý0¤+ _wandbruntime‚218  _runtime‚218‚yü:'¥ŠO QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 8374630837472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8602746 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 9060899269632 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 438481511500 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 9027464850944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3326077 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5286325 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 439645135872 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 451655597568 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚282 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 8587907144704 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 439557706240 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3326079 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1970897 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3326079 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 8587907144704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 8374647876832 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 9027569319936 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5286325 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1433856 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 8609382734848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 451516534784 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1970895 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚499 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8612404 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1434138 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 9061063236096 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024ž»¾1¦ŽO WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1434138 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 9061063236096 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 8374630837472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8602746 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 9060899269632 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 438481511500 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 9027464850944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3326077 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5286325 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 439645135872 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 451655597568 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚282 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 8587907144704 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 439557706240 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3326079 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1970897 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3326079 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 8587907144704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 8374647876832 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 9027569319936 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5286325 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1433856 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 8609382734848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_by–ùqw‚tes.small_pool.freed‚ 451516534784 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1970895 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚499 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8612404%šsKF0§+ _wandbruntime‚218  _runtime‚218y­Êš&¨”M MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5296862 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 8830832837110 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3332722 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 8605090947072 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 8605073907712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 9045511233536 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 440437325824 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 9045615709696 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8629584 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 440437325824 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 9079008287232 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 8626589851648 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚242 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8619912 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5287192 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3411552 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5287192 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3332720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3332720 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 8391388049152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 8391371009792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3332722 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 440524762624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 8626614755328 _step‚38 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858104733731093e+09 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8619912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 439358984722 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚499 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 440524762624 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8629584 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887óűž&©˜M QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1974832 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 452553296896 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 452418435584 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 8605073907712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 9045511233536 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 440437325824 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 9045615709696 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8629584 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 440437325824 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 9079008287232 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 8626589851648 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚242 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8619912 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5287192 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3411552 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5287192 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3332720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3332720 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 8391388049152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 8391371009792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3332722 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 440524762624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 8626614755328 _step‚38 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858104733731093e+09 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8619912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 439358984722 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚499 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 440524762624 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8629584 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5296862 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 8830832837110 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3332722 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 8605090947072 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 9929799136768 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 483553841664 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 496873106432 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 482455778132 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDevicÆ Ò¤¤#eStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3658229 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5814327 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 9469738574848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1577298 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 9211656492832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 9446262334464 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3658227 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5804671 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 9694112270964 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 9929903605760 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2167747 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 482369975008 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2167745 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 9211639453472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858104951200635e+09 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3658229 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 9929903605760 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0{"}È•'¶O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚ 240.311158156 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 9446262334464 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 9211639453472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858104951200635e+09 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3658229 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 9929903605760 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 9929799136768 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 483553841664 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 496873106432 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 482455778132 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3658229 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5814327 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 9469738574848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1577298 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 9211656492832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 9446262334464 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3658227 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5804671 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 9694112270964 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 9929903605760 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2167747 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 482369975008 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2167745 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9489736 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 497777654272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 484520898048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3752069 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5824864 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 497646987264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3751815 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 9984748249600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9480064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚42 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 9228396665152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 9947949995520 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 9947949995520 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 484520898048 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3664872 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5824864 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1580133 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 483333251466 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3664872 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 9463429097472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 9711627074022 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 9711729916618 trainer/global_step‚549ƒNÙ ¹™M J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9489736 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 497777654272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 484520898048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3752069 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5824864 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_dVÏ„,Æevice_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 497646987264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3751815 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 9984748249600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9480064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚42 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 9228396665152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 9947949995520 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 9947949995520 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 484520898048 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3664872 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5824864 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1580133 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 483333251466 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3664872 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 9463429097472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 9711627074022 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 9711729916618 trainer/global_step‚549 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 9947845519360 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5815194 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9489736 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 9463412058112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 9463412058112 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9480064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320*x/Ü0º+ _wandbruntime‚240  _runtime‚240­›„ ÷»ñ /loss/FRt1/assignment_loss‚5.685106759979611 1loss/SRqqt2/assignment_loss‚4.403966975212097 0loss/FBt1/assignment_loss‚1.2798391183217366 /loss/FBt1/detection_loss‚29.524934922655422 _runtime‚ 240.678984288 1loss/SRqqt1/assignment_loss‚4.499621833840462 /loss/FBt2/assignment_loss‚1.126333688244675 1loss/SRqqt1/detection_loss‚2.2118374339419686 .loss/FBt2/detection_loss‚9.516598994885994 epoch‚0 trainer/global_step‚549 $ _timestamp‚1.785810495489185e+09 /loss/FRt2/assignment_loss‚5.875584183669671 0loss/SRqqt2/detection_loss‚7.961906541883946 %loss/total_loss‚79.87589827324845 _step‚43 ,loss/FRt1/detection_loss‚3.8288631031201 /loss/FRt2/detection_loss‚3.9613047174927667£4f`û¼õ ,loss/FRt1/detection_loss‚3.8288631031201 /loss/FRt2/detection_loss‚3.9613047174927667 /loss/FRt1/assignment_loss‚5.685106759979611 1loss/SRqqt2/assignment_loss‚4.403966975212097 0loss/FBt1/assignment_loss‚1.2798391183217366 /loss/FBt1/detection_loss‚29.524934922655422 _runtime‚ 240.678984288 1loss/SRqqt1/assignment_loss‚4.499621833840462 /loss/FBt2/assignment_loss‚1.126333688244675 1loss/SRqqt1/detection_loss‚2.2118374339419686 .loss/FBt2/detection_loss‚9.516598994885994 epoch‚0 trainer/global_step‚549 $ _timestamp‚1.785810495489185e+09 /loss/FRt2/assignment_loss‚5.875584183669671 0loss/SRqqt2/detection_loss‚7.961906541883946 %loss/total_loss‚79.87589827324845 _step‚43+?‰VÚœ½‚(:‘ ÍœÅÓÙÿªƒ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.804' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.93466666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÑJÕ´:¾‚(:¯  ÍœÅÓôÅ̃ disk.drbd1938.in‚0" disk.nvme0n1.out‚ 267.529296875 disk.drbd1931.out‚0 disk.drbd1817.in‚0 disk.rbd1.out‚ 0.9580078125 disk.rbd0.in‚0 disk.drbd1552.out‚0 cpu‚14.747087597855797 disk.drbd1552.in‚0 disk.drbd1938.out‚0 disk.drbd1931.in‚0 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.drbd1584.in‚0 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.drbd2078.out‚0 proc.cpu.threads‚29 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.drbd2041.in‚0% memory_percent‚41.219621896743774 network.sent‚522897 disk.drbd1550.in‚0* proc.memory.percent‚17.172139883041382 disk.drbd1356.in‚0 disk.drbd1356.out‚0 network.recv‚83248 disk.sda1.out‚0 disk.drbd1226.in‚0 disk.sda.out‚8.234375# disk./.usageGB‚587.538158416748 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.sda2.in‚0 disk.drbd1965.in‚0 disk.drbd1285.out‚0* proc.memory.availableMB‚38522.30859375 disk.dm-0.in‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚0 disk.drbd1510.out‚0 disk.sda2.out‚8.234375 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1673.out‚0 disk.sda.in‚0 disk.drbd1226.out‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0$ proc.memory.rssMB‚11253.93359375 disk.drbd1584.out‚0) disk./.usagePercent‚32.85820955244247 disk.dm-0.out‚ 267.529296875O©0¿+ _wandbruntime‚262  _runtime‚262{CÆÀÀ _runtime‚ 262.169830096 _step‚44 ' lr-AdamW/pg1‚1.5437539704698353e-05 ' lr-AdamW/pg2‚1.5437539704698353e-05 trainer/global_step‚599 % _timestamp‚1.7858105169798636e+09#tS/ÊÁÄ _step‚44 ' lr-AdamW/pg1‚1.5437539704698353e-05 ' lr-AdamW/pg2‚1.5437539704698353e-05 trainer/global_step‚599 % _timestamp‚1.7858105169798636e+09 _runtime‚ 262.169830096,jg¿¤0Â+ _wandbruntime‚262  _runtime‚262‘®Q#¥'ßO G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _step‚45 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3990377 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10332708 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6342329 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚10872174580736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚10832133422592 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6342329 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 527637406720 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3990379 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚10330119318528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚10048665108832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858105169808424e+09 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3990379 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 526258438516 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚10304583445504 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0¢éz¿©'Ä£O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚10304583445504 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _step‚45 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3990377 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10332708 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6342329 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚10872174580736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚10832133422592 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6342329 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 527637406720 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3990379 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚10330119318528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚10048665108832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858105169808424e+09 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3990379 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 526258438516-Q¡a0Å+ _wandbruntime‚262  _runtime‚262±j_ŽrÆ©M S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚10850284281344 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10349888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3997020 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚10592524153530 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚10321767247872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocate–dQ=d‚ 528517033472 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4092409 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1723877 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚10890288072192 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚10592626996126 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10340216 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚10347326435328 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 542961636864 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 262.534028286 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6343196 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚10850179805184 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3997022 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 528429596672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10349888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 542826775552 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6352866 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚10890128307200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚10321750208512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚100654052811525Ó¯³&Ç­M QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3997022 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 528429596672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10349888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 542826775552 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6352866 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚10890128307200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚10321750208512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚10065405281152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚10850284281344 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10349888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3997020 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚10592524153530 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚10321767247872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 528517033472 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4092409 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1723877 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚10890288072192 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚10592626996126 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10340216 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚10347326435328 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 542961636864 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 262.534028286 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6343196 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚10850179805184. &g‹0È+ _wandbruntime‚262  _runtime‚262kHöúÉô /loss/FRt1/detection_loss‚3.6281758712583714 0loss/SRqqt2/detection_loss‚6.890081248205641 .loss/FBt1/detection_loss‚27.01344800224671 .loss/FBt2/detection_loss‚8.729348291742749 _runtime‚ 262.534792149 /loss/FRt2/assignment_loss‚5.843631284377154 1loss/SRqqt2/assignment_loss‚4.173658158468164 epoch‚0 trainer/global_step‚599 $ _timestamp‚1.785810517345003e+09 1loss/SRqqt1/assignment_loss‚4.302754952357366 0loss/FBt2/assignment_loss‚0.9133097018514361 $loss/total_loss‚74.4487787940365 /loss/FRt2/detection_loss‚3.8906375285457164 1loss/SRqqt1/detection_loss‚2.4635455866845755 _step‚47 0loss/FRt1/assignment_loss‚5.6239324258954335 0loss/FBt1/assignment_loss‚0.9762557424031771•K4YþÊø /loss/FRt2/detection_loss‚3.8906375285457164 1loss/SRqqt1/detection_loss‚2.4635455866845755 _step‚47 0loss/FRt1/assignment_loss‚5.6239324258954335 0loss/FBt1/assignment_loss‚0.9762557424031771 /loss/FRt1/detection_loss‚3.6281758712583714 0loss/SRqqt2/detection_loss‚6.890081248205641 .loss/FBt1/detection_loss‚27.01344800224671 .loss/FBt2/detection_loss‚8.729348291742749 _runtime‚ 262.534792149 /loss/FRt2/assignment_loss‚5.843631284377154 1loss/SRqqt2/assignment_loss‚4.173658158468164 epoch‚0 trainer/global_step‚599 $ _timestamp‚1.785810517345003e+09 1loss/SRqqt1/assignment_loss‚4.302754952357366 0loss/FBt2/assignment_loss‚0.9133097018514361 $loss/total_loss‚74.4487787940365/Ϙ8Ë‚(:’ ÜœÅÓúú¥ƒ gpu.0.gpu‚17.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚122.986' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚40.995333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0NN^Ç>Ì‚(:³  ÜœÅÓÜ÷׃* disk./.usagePercent‚32.858210619131675 disk.drbd1931.out‚0 disk.drbd1226.out‚0$ proc.memory.rssMB‚11253.93359375 network.recv‚85938 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd1356.in‚0 disk.drbd1289.in‚0 disk.sda2.in‚0 disk.drbd2041.out‚0 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.drbd1356.out‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.dm-0.in‚0 cpu‚14.695043193401181$ disk./.usageGB‚587.5381774902344 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.drbd1510.in‚0 disk.sda1.out‚0 disk.drbd1817.out‚0 disk.drbd1938.out‚0" disk.nvme0n1.out‚ 267.708984375 disk.drbd1931.in‚0 disk.drbd2078.out‚0 proc.cpu.threads‚29 disk.drbd1584.in‚0 disk.sda.in‚0 disk.drbd1285.in‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0$ memory_percent‚41.21981859207153 disk.sda1.in‚0 disk.drbd1226.in‚0 disk.drbd2111.out‚0* proc.memory.percent‚17.172139883041382 disk.drbd1550.out‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.drbd1510.out‚0 network.sent‚567805 disk.drbd1552.in‚0 disk.sda2.out‚ 8.59765625 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.dm-0.out‚ 267.708984375 disk.rbd1.out‚ 0.9580078125 disk.sda.out‚ 8.59765625 disk.drbd2078.in‚0 disk.rbd0.out‚0) proc.memory.availableMB‚ 38522.1796875 disk.drbd2118.in‚0uÎOo0Í+ _wandbruntime‚284  _runtime‚284Ž~~UÅο ' lr-AdamW/pg1‚1.6726149028963658e-05 ' lr-AdamW/pg2‚1.6726149028963658e-05 trainer/global_step‚649 $ _timestamp‚1.785810538931051e+09 _runtime‚ 284.120989199 _step‚48ÑöêöÉÏà _runtime‚ 284.120989199 _step‚48 ' lr-AdamW/pg1‚1.6726149028963658e-05 ' lr-AdamW/pg2‚1.6726149028963658e-05 trainer/global_step‚649 $ _timestamp‚1.785810538931051e+090z¬¤0Ð+ _wandbruntime‚284  _runtime‚284›•¤w¥'ÑŸO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚11190475158528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚11777510671872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚11734467708416 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚11734572177408 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚11162938635264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 571633542144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚11162921595904 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6860675 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚11190450254848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11192860 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _step‚49 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 587060417024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 trainer/global_step‚649 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 571546112512 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4322529 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4426813 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚6860675 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _runtime‚ 284.125656946 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2561445 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6870331 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚11455906429980 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4322527 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6870331 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚11734467708416 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚11734572177408 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚11162938635264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 571633542144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚11162921595904 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6860675 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚11190450254848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11192860 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _step‚49 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 587060417024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 trainer/global_step‚649 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 571546112512 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4322529 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4426813 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚6860675 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _runtime‚ 284.125656946 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2561445 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6870331 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚11455906429980 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4322527 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6870331 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚11752618567168 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 572513168896 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 572425732096 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6880868 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 571024375246 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4329172 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2565382 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11210040 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚11207657371648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚11473524075634 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚11180105398272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚11752618567168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6880868 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚11795626128384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 587968756736 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚649 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚11180088358912 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4433538 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 571110178482 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4433296 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1868154 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360ɉZβ&Õ¬M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2565384 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚11207682275328 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4329172 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4329170 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚242 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚11795779601920 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4329172 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2565382 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11210040 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚11207657371648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚11473524075634 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚11180105398272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚11752618567168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6880868 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚11795626128384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 587968756736 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚649 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚11180088358912 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4433538 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 571110178482 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4433296 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1868154 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚50 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚11752618567168 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 572513168896 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 572425732096 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6880868 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 571024375246 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚12636801994240 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2758297 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7398333 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 615629677568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 614035365532 trainer/global_step‚699 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚12021259746304 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 615629677568 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚12636906463232 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 632376158208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚12636801994240 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4765325 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚11722665301472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4654679 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚12636906463232 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2007271 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2007030 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2758295 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚12021259746304 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚12053012 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 632237095424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚ 305.96364507 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4765568 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 $ _timestamp‚1.785810560769759e+09 _step‚53 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4654677 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚7388677 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 615542247936 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4654679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚12336803509488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 615542247936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚12336700667004 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚12043354 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4654677 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7388677 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚12683043190272 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12050830998528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚11722682340832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚08²eM§'â¡O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 615542247936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚12336700667004 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚12043354 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4654677 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7388677 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚12683043190272 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12050830998528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚11722682340832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚12050806094848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚12636801994240 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2758297 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7398333 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 615629677568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 614035365532 trainer/global_step‚699 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚12021259746304 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 615629677568 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚12636906463232 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 632376158208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚12636801994240 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4765325 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚11722665301472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4654679 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚12636906463232 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2007271 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2007030 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2758295 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚12021259746304 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚12053012 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 632237095424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚ 305.96364507 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4765568 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 $ _timestamp‚1.785810560769759e+09 _step‚53 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4654677 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚7388677 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 615542247936 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4654679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚12336803509488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚12070192 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4772259 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚12354421155142 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12060520 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12070192 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _runtime‚ 306.325739397 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4772012 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚12038426509312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚12038443548672 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2762234 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚12068013211648 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 633145238528 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚12701158450176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚699 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚11739405473792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 614912838754 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7399200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2762232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚12701309826560 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4661320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 616509304320 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2009780 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚12654952852992 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4661322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 616509304320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 616421867520 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 614998641990 _step‚54 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚12654952852992 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4661322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚12068038115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 % _timestamp‚1.7858105611353822e+09 I=Deviß®“ceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4661320 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 616421867520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0÷â³&å­M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚12068038115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 % _timestamp‚1.7858105611353822e+09 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4661320 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 616421867520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚12354318312546 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚12070192 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4772259 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚12354421155142 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12060520 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12070192 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _runtime‚ 306.325739397 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4772012 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚12038426509312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚12038443548672 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2762234 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚12068013211648 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 633145238528 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚12701158450176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚699 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚11739405473792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 614912838754 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7399200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2762232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚12701309826560 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4661320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 616509304320 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2009780 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚12654952852992 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4661322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 616509304320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 616421867520 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 614998641990 _step‚54 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚12654952852992 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4661322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 1871923206”:0æ+ _wandbruntime‚306  _runtime‚306çpŠèûçõ epoch‚0 /loss/FRt1/assignment_loss‚5.735429261181806 2loss/SRqqt2/assignment_loss‚2.9928027942999087 0loss/FBt1/assignment_loss‚0.7005450293421746 0loss/FBt2/assignment_loss‚0.7177437841892242 .loss/FRt2/detection_loss‚3.471410743892193 1loss/SRqqt1/detection_loss‚2.1898685354452865 _runtime‚ 306.326796283 .loss/FBt2/detection_loss‚8.102403779792652 .loss/FRt1/detection_loss‚4.336535948353845 0loss/SRqqt2/detection_loss‚4.692284694358484 trainer/global_step‚699 _step‚55 /loss/FRt2/assignment_loss‚5.906156169607284 2loss/SRqqt1/assignment_loss‚3.3185342201819785 .loss/FBt1/detection_loss‚34.82132050395012 %loss/total_loss‚76.98503546459496 % _timestamp‚1.7858105611366665e+09ÏŠyÿèù 0loss/FBt1/assignment_loss‚0.7005450293421746 0loss/FBt2/assignment_loss‚0.7177437841892242 .loss/FRt2/detection_loss‚3.471410743892193 1loss/SRqqt1/detection_loss‚2.1898685354452865 _runtime‚ 306.326796283 .loss/FBt2/detection_loss‚8.102403779792652 .loss/FRt1/detection_loss‚4.336535948353845 0loss/SRqqt2/detection_loss‚4.692284694358484 trainer/global_step‚699 _step‚55 /loss/FRt2/assignment_loss‚5.906156169607284 2loss/SRqqt1/assignment_loss‚3.3185342201819785 .loss/FBt1/detection_loss‚34.82132050395012 %loss/total_loss‚76.98503546459496 % _timestamp‚1.7858105611366665e+09 epoch‚0 /loss/FRt1/assignment_loss‚5.735429261181806 2loss/SRqqt2/assignment_loss‚2.99280279429990877Só×b‘é‚(:† ‰ÅÓ°ñø‚ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚105.372' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚35.124 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0>¢w¼;ê‚(:°  ‰ÅÓ‹¨Áƒ disk.drbd1552.in‚0 disk.sda2.out‚9.796875 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.drbd1285.in‚0% memory_percent‚41.220784187316895 disk.drbd2078.in‚0 disk.drbd1965.out‚0 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.rbd1.out‚ 1.146484375 disk.drbd1584.out‚0) disk./.usagePercent‚32.85820955244247# disk./.usageGB‚587.538158416748 disk.drbd1673.out‚0 disk.drbd2041.in‚0 disk.drbd2111.out‚0* proc.memory.percent‚17.172139883041382 disk.drbd1938.out‚0 disk.drbd1289.in‚0 disk.sda1.in‚0 disk.drbd1673.in‚0 disk.drbd1931.in‚0 disk.sda.out‚9.796875 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.drbd1510.in‚0 disk.drbd1356.in‚0! disk.dm-0.out‚269.39599609375 disk.drbd2118.out‚0 network.recv‚92336 disk.drbd1938.in‚0 disk.sda2.in‚0 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.drbd2111.in‚0 disk.sda.in‚0 disk.drbd1965.in‚0( proc.memory.availableMB‚ 38521.546875 disk.drbd1817.in‚0 network.sent‚661077 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.drbd1510.out‚0 cpu‚14.569015294521499 disk.drbd1828.out‚0 disk.dm-0.in‚0 proc.cpu.threads‚29 disk.drbd2118.in‚0 disk.nvme0n1.in‚0$ disk.nvme0n1.out‚269.39599609375 disk.drbd2078.out‚0 disk.drbd1289.out‚0 disk.sda1.out‚0$ proc.memory.rssMB‚11253.93359375 disk.rbd0.in‚0\þ[0ë+ _wandbruntime‚328  _runtime‚328ö9·¹ÆìÀ ' lr-AdamW/pg2‚1.9303367677494267e-05 trainer/global_step‚749 % _timestamp‚1.7858105828611386e+09 _runtime‚ 328.051096739 _step‚56 ' lr-AdamW/pg1‚1.9303367677494267e-05ë¹ÌAÊíÄ ' lr-AdamW/pg1‚1.9303367677494267e-05 ' lr-AdamW/pg2‚1.9303367677494267e-05 trainer/global_step‚749 % _timestamp‚1.7858105828611386e+09 _runtime‚ 328.051096739 _step‚568¡|L0î+ _wandbruntime‚328  _runtime‚328.$Ф'ïžO Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚13539136280064 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 659625812992 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4986827 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2955145 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4986827 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2150965 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7926335 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 659538383360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚12879597896704 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2150698 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚13588609965056 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚13539240749056 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚12559673917472 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚13539136280064 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12911186838528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 trainer/global_step‚749 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚12879597896704 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5106112 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚13217597746512 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 659538383360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4986829 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚12559690956832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7926335 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚269 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5105843 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 677448030208 _step‚57 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚13539240749056 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 677582898688 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚13217700588996 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 658009632164 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚7916679 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 659625812992 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚13539136280064 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 659625812992 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4986827 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2955145 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4986827 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2150965 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7926335 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 659538383360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚12879597896704 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2150698 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚13588609965056 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚13539240749056 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚12559673917472 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.curren¨q‘Ót‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚13539136280064 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12911186838528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 trainer/global_step‚749 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚12879597896704 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5106112 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚13217597746512 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 659538383360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4986829 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚12559690956832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚09d´U0ñ+ _wandbruntime‚328  _runtime‚328¶£Ò®&ò¨M H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4993472 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚12896764659712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12930344 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚243 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 678358780416 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚12896764659712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12920672 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚749 _runtime‚ 328.419989978 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4993470 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4993470 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2153571 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚13606727832064 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚12576414089792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2959082 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 660418002944 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚12928393955328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚12930344 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚58 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚13235215392054 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚13557182662656 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚12896781699072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 $ _timestamp‚1.785810583229635e+09 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12920672 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚13557287138816 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5112653 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 658887105498 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 658801302262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚13606877111296 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 660418002944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 678483155968 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7936872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7927202 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚12928369051648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚12896764659712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12930344 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚243 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 678358780416 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚12896764659712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12920672 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚749 _runtime‚ 328.419989978 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4993470 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4993470 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2153571 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚13606727832064 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚12576414089792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2959082 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 660418002944 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚12928393955328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚12930344 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚58 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚13235215392054 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚13557182662656 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚12896781699072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 $ _timestamp‚1.785810583229635e+09 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12920672 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚13557287138816 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5112653 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 658887105498 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 658801302262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚13606877111296 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 660418002944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 678483155968 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7936872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7927202 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚12928369051648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0:Ô”ïB0ô+ _wandbruntime‚328  _runtime‚328 ÄBûõõ 0loss/FRt1/assignment_loss‚5.7628815631673795 .loss/FRt2/assignment_loss‚5.90657928545181 .loss/FRt1/detection_loss‚3.074796387944559 0loss/SRqqt2/detection_loss‚5.732544066960162 .loss/FBt1/detection_loss‚31.48249634287574 2loss/SRqqt1/assignment_loss‚2.9150169787516123 1loss/FBt2/assignment_loss‚0.34742817102835094 trainer/global_step‚749 _step‚59 1loss/SRqqt1/detection_loss‚1.9671664867934038 % _timestamp‚1.7858105832309606e+09 1loss/SRqqt2/assignment_loss‚2.663424095240506 0loss/FBt1/assignment_loss‚0.7342240105975758 .loss/FRt2/detection_loss‚4.508222914721868 .loss/FBt2/detection_loss‚9.040674419637455 %loss/total_loss‚74.13545472317043 epoch‚0 _runtime‚ 328.420747033¡¥ÿöù _step‚59 1loss/SRqqt1/detection_loss‚1.9671664867934038 % _timestamp‚1.7858105832309606e+09 1loss/SRqqt2/assignment_loss‚2.663424095240506 0loss/FBt1/assignment_loss‚0.7342240105975758 .loss/FRt2/detection_loss‚4.508222914721868 .loss/FBt2/detection_loss‚9.040674419637455 %loss/total_loss‚74.13545472317043 epoch‚0 _runtime‚ 328.420747033 0loss/FRt1/assignment_loss‚5.7628815631673795 .loss/FRt2/assignment_loss‚5.90657928545181 .loss/FRt1/detection_loss‚3.074796387944559 0loss/SRqqt2/detection_loss‚5.732544066960162 .loss/FBt1/detection_loss‚31.48249634287574 2loss/SRqqt1/assignment_loss‚2.9150169787516123 1loss/FBt2/assignment_loss‚0.34742817102835094 trainer/global_step‚749;·1Õ˜œ÷‚(:‘ ˜ÅÓˆ¼¢ƒ gpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚123.533' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.17766666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ô-ßì=ø‚(:²  ˜ÅÓÍüă* proc.memory.percent‚17.172139883041382 proc.cpu.threads‚29 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.dm-0.in‚0 disk.drbd1510.in‚0 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd2041.out‚0 disk.drbd1285.out‚0 disk.drbd1828.out‚0 network.sent‚706067$ disk./.usageGB‚587.5381774902344 disk.drbd2111.out‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.rbd1.out‚ 1.146484375 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.rbd0.in‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 10.41796875 disk.sda1.out‚0 disk.drbd1552.in‚0 cpu‚14.533305632724742* disk./.usagePercent‚32.858210619131675 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd1285.in‚0 disk.drbd1931.in‚0 disk.sda.in‚0 disk.drbd2078.in‚0 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd1550.out‚0 disk.sda1.in‚0 disk.drbd2118.out‚0 disk.drbd1226.out‚0 disk.drbd1828.in‚0 disk.sda2.in‚0# memory_percent‚41.2205696105957 disk.rbd0.out‚0 disk.dm-0.out‚269.6513671875$ proc.memory.rssMB‚11253.93359375 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.nvme0n1.in‚0 disk.sda.out‚ 10.41796875 disk.drbd1510.out‚0 disk.drbd1289.in‚0# disk.nvme0n1.out‚269.6513671875& proc.memory.availableMB‚ 38521.6875 network.recv‚94900 disk.drbd1356.out‚0 disk.drbd1550.in‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0 disk.drbd1584.out‚0 disk.drbd1673.out‚0Æc$žù‚(:’ §ÅÓòØŽƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚120.791' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚40.263666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0øà¬Cú‚(:¸  §ÅÓßùÖƒ disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.drbd1226.in‚0 disk.drbd2111.in‚0 disk.rbd1.in‚0 disk.drbd1965.in‚0 disk.dm-0.in‚0 disk.drbd1673.out‚0 disk.drbd1584.in‚0 disk.drbd1938.in‚0 disk.rbd1.out‚ 1.474609375 disk.drbd1552.out‚0 disk.drbd2118.out‚0# memory_percent‚41.2209689617157 cpu‚14.500519212301278 network.recv‚97517 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.drbd1938.out‚0 network.sent‚751212$ disk./.usageGB‚587.7904243469238 disk.drbd1550.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd1356.out‚0 disk.drbd1965.out‚0 disk.sda2.in‚0 disk.drbd1285.in‚0 disk.sda1.out‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.drbd1817.out‚0 proc.cpu.threads‚29 disk.drbd1828.in‚0 disk.drbd1552.in‚0$ disk.nvme0n1.out‚528.65869140625 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.sda2.out‚ 10.75390625 disk.nvme0n1.in‚0 disk.sda.out‚ 10.75390625* disk./.usagePercent‚32.872317583857814 disk.drbd1356.in‚0$ proc.memory.rssMB‚11253.93359375 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd1828.out‚0! disk.dm-0.out‚528.65869140625 disk.drbd1289.out‚0 disk.sda1.in‚0* proc.memory.percent‚17.172139883041382 disk.sda.in‚0 disk.drbd2111.out‚0* proc.memory.availableMB‚38521.42578125™G0û+ _wandbruntime‚349  _runtime‚349neD-Äü¾ _step‚60 & lr-AdamW/pg1‚2.059197700175957e-05 & lr-AdamW/pg2‚2.059197700175957e-05 trainer/global_step‚799 % _timestamp‚1.7858106040884428e+09 _runtime‚ 349.278405772,žóÈý & lr-AdamW/pg1‚2.059197700175957e-05 & lr-AdamW/pg2‚2.059197700175957e-05 trainer/global_step‚799 % _timestamp‚1.7858106040884428e+09 _runtime‚ 349.278405772 _step‚60<)_«0þ+ _wandbruntime‚349  _runtime‚349BYs±^ÿŸO ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚14494351911424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3151997 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2294248 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13763658 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 703621948416 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8444681 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5446243 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 701812292548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚14441470565888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚13771517774848 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 722809232896 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚13737953086464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 _step‚61 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13773316 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚14494192139264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5318979 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚14441470565888 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5318979 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚13396699572832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 703534518784 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚14098597668504 F@DeviceStatsMonitor.on_train_batch_start/res& ÅWGerved_bytes.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8454337 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚13737936047104 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 701898095672 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3151995 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 703621948416 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 722674364416 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚799 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2294515 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚14098494826020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚13737936047104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚14441575034880 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚267 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5318977 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚13396682533472 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5318977 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5446512 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚13737953086464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚13771542678528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 703534518784 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8454337 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0Õ¢çê©'€£O UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚14441575034880 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8444681 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚269 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7858106040894632e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚13771542678528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 703534518784 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8454337 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚14494351911424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3151997 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2294248 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13763658 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 703621948416 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8444681 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5446243 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 701812292548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚14441470565888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚13771517774848 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 722809232896 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚13737953086464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 _step‚61 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13773316 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚14494192139264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5318979 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚14441470565888 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5318979 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚13396699572832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 703534518784 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚14098597668504 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8454337 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚13737936047104 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 701898095672 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3151995 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 703621948416 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 722674364416 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚799 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2294515 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚14098494826020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚13737936047104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚14441575034880 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚267 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5318977 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚13396682533472 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5318977 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5446512 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚13737953086464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5325622 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13790496 _runtime‚ 349.63737409 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13780824 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5325620 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7858106044469993e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 704414138368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚14512461814784 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚14116112471562 _step‚62 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 723577158144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚13755102810112 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8455204 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚13755119849472 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚13413422705792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3155934 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚14459621424640 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚13413439745152 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚14459516948480 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3155932 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5325622 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚13788749795328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2297139 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5453071 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0·›xƒ¬M SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚14116215314158 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8464874 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5453071 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 704501575168 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13780824 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5325620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚241 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚239 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 trainer/global_step‚799 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8455204 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚13755102810112 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2297378 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚14459621424640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5325622 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13790496 _runtime‚ 349.63737409 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13780824 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5325620 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7858106044469993e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 704414138368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚14512461814784 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚14116112471562 _step‚62 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 723577158144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚13755102810112 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8455204 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚13755119849472 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚13413422705792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3155934 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚14459621424640 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.laõÁÇÈ£ rge_pool.allocated‚13413439745152 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚14459516948480 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3155932 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5325622 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚13788749795328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2297139 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 Eš…†s0„+ _wandbruntime‚349  _runtime‚349a°µBú…ô _step‚63 .loss/FRt2/assignment_loss‚5.83218534996635 .loss/FRt1/detection_loss‚4.031310509890318 epoch‚0 $ _timestamp‚1.785810604448295e+09 .loss/FBt2/detection_loss‚6.802608220294485 %loss/total_loss‚71.31353379946961 1loss/SRqqt1/assignment_loss‚2.852159555753072 2loss/SRqqt2/assignment_loss‚2.6472500259593383 1loss/FBt2/assignment_loss‚0.27106312722625103 .loss/FRt2/detection_loss‚4.258027402586059 0loss/SRqqt2/detection_loss‚5.413175076246262 trainer/global_step‚799 0loss/FRt1/assignment_loss‚5.4400447607040405 0loss/FBt1/assignment_loss‚0.6874460957267068 1loss/SRqqt1/detection_loss‚1.8333304708993334 .loss/FBt1/detection_loss‚31.24493320421739 _runtime‚ 349.638425747ÛÓ§%þ†ø 1loss/SRqqt1/detection_loss‚1.8333304708993334 .loss/FBt1/detection_loss‚31.24493320421739 _runtime‚ 349.638425747 _step‚63 .loss/FRt2/assignment_loss‚5.83218534996635 .loss/FRt1/detection_loss‚4.031310509890318 epoch‚0 $ _timestamp‚1.785810604448295e+09 .loss/FBt2/detection_loss‚6.802608220294485 %loss/total_loss‚71.31353379946961 1loss/SRqqt1/assignment_loss‚2.852159555753072 2loss/SRqqt2/assignment_loss‚2.6472500259593383 1loss/FBt2/assignment_loss‚0.27106312722625103 .loss/FRt2/detection_loss‚4.258027402586059 0loss/SRqqt2/detection_loss‚5.413175076246262 trainer/global_step‚799 0loss/FRt1/assignment_loss‚5.4400447607040405 0loss/FBt1/assignment_loss‚0.6874460957267068?ãì¢A›‡‚(: ¶ÅÓÂ˃ gpu.0.gpu‚30.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚122.4' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚40.800000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ðG¨Dˆ‚(:¹  ¶ÅÓ…”Úƒ disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.rbd1.out‚ 1.474609375 disk.drbd1285.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.drbd1938.out‚0 disk.rbd0.in‚0* proc.memory.availableMB‚38521.30859375 disk.sda2.out‚ 11.04296875$ disk.nvme0n1.out‚528.93994140625 disk.drbd2041.in‚0 disk.sda1.in‚0 disk.sda.in‚0 disk.drbd1584.out‚0 disk.sda.out‚ 11.04296875! disk.dm-0.out‚528.93994140625 disk.drbd1289.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0* proc.memory.percent‚17.172139883041382 disk.drbd2111.in‚0$ proc.memory.rssMB‚11253.93359375 disk.drbd1356.out‚0 disk.drbd1828.in‚0 proc.cpu.threads‚29 network.sent‚754408 disk.rbd1.in‚0 disk.drbd1356.in‚0 disk.drbd2078.in‚0 disk.dm-0.in‚0) disk./.usagePercent‚32.87234382441223 disk.drbd1584.in‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd1965.in‚0 disk.drbd1289.out‚0 disk.drbd2111.out‚0 disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 network.recv‚98696$ disk./.usageGB‚587.7908935546875 disk.drbd1938.in‚0 disk.drbd1931.out‚0 disk.drbd1285.in‚0 disk.drbd1510.out‚0% memory_percent‚41.221147775650024 disk.drbd2041.out‚0 cpu‚14.469475443611975>y¾0‰+ _wandbruntime‚371  _runtime‚371…èù>ÆŠÀ trainer/global_step‚849 % _timestamp‚1.7858106259345474e+09 _runtime‚ 371.124499801 _step‚64 ' lr-AdamW/pg1‚2.1880586326024876e-05 ' lr-AdamW/pg2‚2.1880586326024876e-05xÊ‹Ä trainer/global_step‚849 % _timestamp‚1.7858106259345474e+09 _runtime‚ 371.124499801 _step‚64 ' lr-AdamW/pg1‚2.1880586326024876e-05 ' lr-AdamW/pg2‚2.1880586326024876e-05@ŽYR 0Œ+ _wandbruntime‚371  _runtime‚3713>‡¿¥'ŸO XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚15343909320704 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5787875 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 747530654208 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14633468 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2438774 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3348847 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚15343804851712 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 745700756056 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8982339 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚15399916882432 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚14233691149472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5787619 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚14631873614848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _runtime‚ 371.125730979 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5651129 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 747618083840 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8982339 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚14979391905528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5651129 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚15343909320704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2439028 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 767881398272 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚14233708188832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 747530654208 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14633468 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2438774 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3348847 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚15343804851712 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 745700756056 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8982339 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚15399916882432 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚14233691149472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5787619 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚14631873614848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _runtime‚ 371.125730979 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5651129 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 747618083840 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8982339 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚14979391905528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5651129 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚15343909320704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2439028 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 767881398272 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚14233708188832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2441601 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚14997009551070 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8992876 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚14613440960512 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3352782 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5657772 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚15361955710464 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚14997112393666 $ _timestamp‚1.785810626296175e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8992876 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚66 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8983206 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2441844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5794383 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚14250431321792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚15361955710464 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 748497710592 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 768784224768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚849 _runtime‚ 371.486502881 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14650648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚15361851234304 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5657770 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚14649080731648 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 746664032514 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14650648 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚14613457999872 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚15418022624256 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚14250448361152 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14640976 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚14649105635328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚15417864956416 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚14613440960512 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14640976 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5657770 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2441601 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚14997009551070 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8992876 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚14613440960512 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3352782 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5657772 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚15361955710464 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚14997112393666 $ _timestamp‚1.785810626296175e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8992876 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚66 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8983206 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2441844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5794383 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚14250431321792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚15361955710464 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 748497710592 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 768784224768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚849 _runtime‚ 371.486502881 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14650648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚15361851234304 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5657770 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚14649080731648 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 746664032514 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14650648 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚14613457999872 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚15418022624256 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚14250448361152 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14640976 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚14649105635328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚15417864956416 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251B.š:}0’+ _wandbruntime‚371  _runtime‚371Y%yú“ô 1loss/SRqqt2/assignment_loss‚2.702065600624567 .loss/FRt2/detection_loss‚3.875911746039448 % _timestamp‚1.7858106262974248e+09 _runtime‚ 371.487210899 1loss/SRqqt1/assignment_loss‚2.612418702056816 0loss/SRqqt1/detection_loss‚2.104683409655794 .loss/FBt1/detection_loss‚43.68713872693479 epoch‚0 /loss/FRt1/assignment_loss‚5.614925384521484 /loss/FRt2/assignment_loss‚5.797788798090923 0loss/FBt1/assignment_loss‚0.5989321619272232 1loss/FBt2/assignment_loss‚0.11769175295318876 .loss/FRt1/detection_loss‚4.257896916270256 %loss/total_loss‚81.49513964702273 trainer/global_step‚849 _step‚67 1loss/SRqqt2/detection_loss‚3.9954161466676976 .loss/FBt2/detection_loss‚6.1302703012805447À@¼þ”ø /loss/FRt1/assignment_loss‚5.614925384521484 /loss/FRt2/assignment_loss‚5.797788798090923 0loss/FBt1/assignment_loss‚0.5989321619272232 1loss/FBt2/assignment_loss‚0.11769175295318876 .loss/FRt1/detection_loss‚4.257896916270256 %loss/total_loss‚81.49513964702273 trainer/global_step‚849 _step‚67 1loss/SRqqt2/detection_loss‚3.9954161466676976 .loss/FBt2/detection_loss‚6.130270301280544 1loss/SRqqt2/assignment_loss‚2.702065600624567 .loss/FRt2/detection_loss‚3.875911746039448 % _timestamp‚1.7858106262974248e+09 _runtime‚ 371.487210899 1loss/SRqqt1/assignment_loss‚2.612418702056816 0loss/SRqqt1/detection_loss‚2.104683409655794 .loss/FBt1/detection_loss‚43.68713872693479 epoch‚0C ($<•‚(:’ ÅÅÓÊ‹ ƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚100.049' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚33.349666666666664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ú—Ê×=–‚(:²  ÅÅÓÈœ´ƒ disk.drbd2111.out‚0 disk.drbd1817.in‚0 disk.drbd2041.out‚0 disk.sda.out‚ 11.640625 disk.drbd1965.in‚0 disk.drbd1550.in‚0( disk./.usagePercent‚32.8723436110744 disk.sda1.in‚0 disk.rbd1.out‚ 1.5068359375 disk.drbd1965.out‚0* proc.memory.percent‚17.172139883041382# memory_percent‚41.2212610244751 network.sent‚799651 disk.sda2.in‚0 disk.drbd1584.in‚0 disk.rbd0.in‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.drbd1931.out‚0$ disk./.usageGB‚587.7908897399902 disk.drbd1510.out‚0 disk.rbd1.in‚0 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.drbd1289.in‚0 disk.drbd1817.out‚0( proc.memory.availableMB‚ 38521.234375 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚0 disk.drbd2118.in‚0 disk.sda.in‚0 cpu‚14.439723602672053! disk.dm-0.out‚529.21826171875 disk.drbd1356.out‚0 disk.drbd2111.in‚0 network.recv‚101413 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd1285.in‚0 disk.drbd1510.in‚0 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd1226.in‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.sda2.out‚ 11.640625 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.drbd1938.out‚0 disk.drbd1289.out‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚0$ disk.nvme0n1.out‚529.21826171875 disk.drbd1550.out‚0 disk.drbd1931.in‚0$ proc.memory.rssMB‚11253.93359375 proc.cpu.threads‚29H¨t‘—‚(:† ÔÅÓÀ®†ƒ gpu.0.gpu‚22.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚118.311' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚39.437 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ýÊS˜‚(:È  ÔÅÓ©ÉɃ disk.drbd1289.out‚0 disk.nvme0n1.in‚0 disk.sda1.out‚0 disk.sda2.out‚ 12.90234375 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.drbd2111.in‚0 disk.drbd1226.out‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1817.in‚0 disk.drbd1673.out‚0 disk.sda2.in‚ 0.00390625 network.sent‚844719 network.recv‚104069 disk.drbd1965.out‚0 disk.drbd1289.in‚0 disk.drbd1965.in‚0! disk.dm-0.out‚529.95263671875 disk.drbd1552.out‚0$ proc.memory.rssMB‚11253.93359375 disk.drbd1510.in‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0) disk./.usagePercent‚32.87233721093917$ disk.nvme0n1.out‚529.95263671875 disk.sda.out‚ 12.90234375 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd2118.out‚0 disk.sda.in‚ 0.00390625 disk.drbd2078.out‚0 disk.drbd1584.in‚0 disk.drbd2041.in‚0$ disk./.usageGB‚587.7907752990723 disk.drbd2078.in‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0& proc.memory.availableMB‚ 38520.9375 disk.rbd0.out‚0 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.drbd1931.in‚0 proc.cpu.threads‚29 disk.drbd1356.in‚0 disk.drbd1931.out‚0$ memory_percent‚41.22171401977539 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚0 disk.drbd2041.out‚0* proc.memory.percent‚17.172139883041382 disk.drbd1938.out‚0 disk.rbd1.out‚ 1.5068359375 disk.sda1.in‚0 disk.drbd1584.out‚0 cpu‚14.413842126506776Ña¿†0™+ _wandbruntime‚392  _runtime‚392o†¤ÆšÀ ' lr-AdamW/pg1‚2.3169195650290183e-05 ' lr-AdamW/pg2‚2.3169195650290183e-05 trainer/global_step‚899 % _timestamp‚1.7858106473067589e+09 _runtime‚ 392.496721046 _step‚68B1ìTÊ›Ä ' lr-AdamW/pg1‚2.3169195650290183e-05 ' lr-AdamW/pg2‚2.3169195650290183e-05 trainer/global_step‚899 % _timestamp‚1.7858106473067589e+09 _runtime‚ 392.496721046 _step‚68DaAS‘0œ+ _wandbruntime‚392  _runtime‚392WKÏ¥'ŸO @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9500685 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5983277 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚15070716804832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7858106473077338e+09 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 392.497924945 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15493620 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 789675022688 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 _step‚69 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5983279 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6128420 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚15492254358528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚16246139137536 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚15454629387264 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6128161 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚16305511479808 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15483962 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚16246139137536 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3545695 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 791614219264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9500685 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5983279 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.currenž÷âÃgt‚259 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 789675022688 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 _step‚69 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5983279 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6128420 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚15492254358528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚16246139137536 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚15454629387264 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6128161 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚16305511479808 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15483962 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚16246139137536 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3545695 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 791614219264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9500685 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5983279EÎ\Ø*0Ÿ+ _wandbruntime‚392  _runtime‚392¤z˜¸¯& ©M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚15878009473174 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚15501128 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3549632 % _timestamp‚1.7858106476718078e+09 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5989920 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6134948 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5989922 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9511208 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚15087456977152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 814159238144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 792493846016 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3549634 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15510800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚15509461475328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚70 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 814030668288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9520878 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚16323620713472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚15471779110912 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚16264289996288 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚15471779110912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15501128 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9511208 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚899 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚16264289996288 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 792493846016 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 792406409216 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9520878 _runtime‚ 392.862131969 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚15471796150272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2585570 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2585316 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 790552496022 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 790466692786 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5989922 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚16323467239936‚·Å ³&¡­M MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9511208 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚899 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚16264289996288 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 792493846016 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 792406409216 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9520878 _runtime‚ 392.862131969 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚15471796150272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2585570 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2585316 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 790552496022 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 790466692786 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5989922 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚16323467239936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚15878009473174 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚15501128 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3549632 % _timestamp‚1.7858106476718078e+09 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5989920 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6134948 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5989922 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9511208 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚15087456977152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 814159238144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 792493846016 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3549634 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15510800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚15509461475328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚70 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 814030668288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9520878 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚16323620713472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚15471779110912 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚16264289996288 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚15471779110912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15501128FëÐp£0¢+ _wandbruntime‚392  _runtime‚392þ9¼n£ú 1loss/SRqqt2/assignment_loss‚2.729372472532334 1loss/FBt2/assignment_loss‚0.08604205656974089 1loss/SRqqt1/detection_loss‚1.8293135493939998 0loss/FRt2/assignment_loss‚5.7189045050709515 /loss/FRt2/detection_loss‚3.3133676184206893 epoch‚0 /loss/FRt1/assignment_loss‚5.546965169317929 2loss/SRqqt1/assignment_loss‚2.3523665997717114 /loss/FBt2/detection_loss‚13.081194250578418 %loss/total_loss‚87.31268313249583 trainer/global_step‚899 _runtime‚ 392.862878616 0loss/FBt1/assignment_loss‚0.6504343934357166 /loss/FRt1/detection_loss‚3.9225760996341705 0loss/SRqqt2/detection_loss‚5.132058320987609 /loss/FBt1/detection_loss‚42.950088096782565 % _timestamp‚1.7858106476730738e+09 _step‚715 K¤þ 0loss/FRt2/assignment_loss‚5.7189045050709515 /loss/FRt2/detection_loss‚3.3133676184206893 epoch‚0 /loss/FRt1/assignment_loss‚5.546965169317929 2loss/SRqqt1/assignment_loss‚2.3523665997717114 /loss/FBt2/detection_loss‚13.081194250578418 %loss/total_loss‚87.31268313249583 trainer/global_step‚899 _runtime‚ 392.862878616 0loss/FBt1/assignment_loss‚0.6504343934357166 /loss/FRt1/detection_loss‚3.9225760996341705 0loss/SRqqt2/detection_loss‚5.132058320987609 /loss/FBt1/detection_loss‚42.950088096782565 % _timestamp‚1.7858106476730738e+09 _step‚71 1loss/SRqqt2/assignment_loss‚2.729372472532334 1loss/FBt2/assignment_loss‚0.08604205656974089 1loss/SRqqt1/detection_loss‚1.8293135493939998Giç"›¥‚(: ãÅÓ€ƒ‰ƒ gpu.0.gpu‚33.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚125.15' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.71666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0žBÑQ¦‚(:Æ  ãÅÓø–Öƒ) disk./.usagePercent‚32.87233742427701 disk.rbd0.out‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.sda1.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd2041.in‚0 disk.drbd1510.in‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.drbd1828.in‚0 disk.drbd1584.in‚0 disk.drbd1550.out‚0* proc.memory.percent‚17.172139883041382 disk.sda.out‚ 13.171875 disk.drbd1226.in‚0 network.sent‚847959 disk.drbd2078.in‚0 disk.rbd1.out‚ 1.6630859375 proc.cpu.threads‚29 disk.rbd0.in‚0 disk.drbd1552.in‚0 disk.drbd1356.out‚0 disk.drbd1285.in‚0 disk.sda.in‚ 0.00390625 disk.drbd1550.in‚0 disk.sda1.in‚0 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd1673.in‚0) proc.memory.availableMB‚ 38520.8828125 network.recv‚105314 disk.sda2.in‚ 0.00390625 disk.drbd1965.out‚0 disk.drbd2111.out‚0 disk.drbd1938.out‚0 disk.drbd1356.in‚0 disk.drbd1285.out‚0 disk.dm-0.out‚530.3466796875 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.drbd1510.out‚0 disk.sda2.out‚ 13.171875 disk.dm-0.in‚0 disk.drbd1289.in‚0 disk.nvme0n1.in‚0# disk.nvme0n1.out‚530.3466796875 cpu‚14.387487660708235 disk.drbd1828.out‚0 disk.drbd1938.in‚0% memory_percent‚41.221797466278076$ disk./.usageGB‚587.7907791137695 disk.drbd1226.out‚0 disk.drbd2111.in‚0 disk.drbd1289.out‚0$ proc.memory.rssMB‚11253.93359375’‰ù0§+ _wandbruntime‚414  _runtime‚414me[JŨ¿ trainer/global_step‚949 % _timestamp‚1.7858106688414576e+09 _runtime‚ 414.03141653 _step‚72 ' lr-AdamW/pg1‚2.4457804974555488e-05 ' lr-AdamW/pg2‚2.4457804974555488e-05ÄÐÉ©Ã ' lr-AdamW/pg1‚2.4457804974555488e-05 ' lr-AdamW/pg2‚2.4457804974555488e-05 trainer/global_step‚949 % _timestamp‚1.7858106688414576e+09 _runtime‚ 414.03141653 _step‚72HåO0ª+ _wandbruntime‚414  _runtime‚414Ù^ë—«¢O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚17148473423360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚16352585294848 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 858364686336 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚73 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚15907725420832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 835610354688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚16741288907028 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0f;̬'¬¦O ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚73 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚15907725420832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 835610354688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚16741288907028 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚17148473423360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚16352585294848 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 858364686336IzÓQÖ0­+ _wandbruntime‚414  _runtime‚414gKn³&®­M E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚949 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6476741 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚16330117261312 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16361280 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 % _timestamp‚1.7858106691999588e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2730506 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚16758803710086 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _step‚74 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 414.390259397 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16370952 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚17166624282112 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10039210 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10039210 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 836402544640 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 836489981440 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚17229061374464 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚16369817315328 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6322072 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6322070 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6476990 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚16330134300672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16361280 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚17229219042304 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚16369792411648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 859268962816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 834440959530 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10048880 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2730259 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 859401726976 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚16330134300672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚249 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚247 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6322070 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 836489981440 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10048880 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3746484 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6322072 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16370952 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚17166519805952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3746482 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596~ièm= ¯±M H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚249 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚247 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6322070 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 836489981440 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10048880 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3746484 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6322072 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16370952 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚17166519805952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3746482 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitoC€}zr.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚949 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6476741 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚16330117261312 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16361280 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 % _timestamp‚1.7858106691999588e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2730506 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚16758803710086 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _step‚74 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 414.390259397 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16370952 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚17166624282112 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10039210 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10039210 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 836402544640 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 836489981440 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚17229061374464 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚16369817315328 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6322072 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6322070 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6476990 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚16330134300672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16361280 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚17229219042304 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚16369792411648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 859268962816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 834440959530 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10048880 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2730259 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 859401726976 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚16330134300672Ju ²0°+ _wandbruntime‚414  _runtime‚414k>_sü±ö 2loss/SRqqt1/assignment_loss‚2.4079474136892673 1loss/FBt2/assignment_loss‚0.13208849822650803 .loss/FBt2/detection_loss‚5.578191439196431 %loss/total_loss‚79.19554029916452 _runtime‚ 414.39142126 .loss/FRt2/detection_loss‚3.832429357227825 .loss/FBt1/detection_loss‚42.79326028563082 $ _timestamp‚1.785810669201186e+09 0loss/FRt1/assignment_loss‚5.5957795056429775 /loss/SRqqt2/detection_loss‚4.64373167648035 epoch‚0 trainer/global_step‚949 0loss/FRt2/assignment_loss‚5.6244435367130095 2loss/SRqqt2/assignment_loss‚2.5615358983769134 0loss/FBt1/assignment_loss‚0.7465800791978836 /loss/FRt1/detection_loss‚3.6299988415769557 1loss/SRqqt1/detection_loss‚1.6495537672055869 _step‚75*‰£ª²ú _runtime‚ 414.39142126 .loss/FRt2/detection_loss‚3.832429357227825 .loss/FBt1/detection_loss‚42.79326028563082 $ _timestamp‚1.785810669201186e+09 0loss/FRt1/assignment_loss‚5.5957795056429775 /loss/SRqqt2/detection_loss‚4.64373167648035 epoch‚0 trainer/global_step‚949 0loss/FRt2/assignment_loss‚5.6244435367130095 2loss/SRqqt2/assignment_loss‚2.5615358983769134 0loss/FBt1/assignment_loss‚0.7465800791978836 /loss/FRt1/detection_loss‚3.6299988415769557 1loss/SRqqt1/detection_loss‚1.6495537672055869 _step‚75 2loss/SRqqt1/assignment_loss‚2.4079474136892673 1loss/FBt2/assignment_loss‚0.13208849822650803 .loss/FBt2/detection_loss‚5.578191439196431 %loss/total_loss‚79.19554029916452KøK,¿œ³‚(:‘ òÅÓŒ¸™ƒ gpu.0.gpu‚41.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚118.841' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚39.61366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ø•%éP´‚(:Å  òÅÓ—£²ƒ disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.drbd2041.out‚0 disk.dm-0.in‚0 disk.sda1.in‚0# disk.nvme0n1.out‚530.7763671875* proc.memory.availableMB‚38522.00390625) proc.memory.percent‚17.17214584350586 disk.drbd1226.out‚0 disk.drbd1289.in‚0 disk.drbd2111.out‚0 disk.drbd1356.out‚0 disk.sda2.in‚ 0.00390625 disk.nvme0n1.in‚0) disk./.usagePercent‚32.87233742427701 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.drbd1673.in‚0 network.sent‚892993 disk.drbd1550.in‚0 disk.drbd2118.in‚0 disk.drbd1356.in‚0 disk.drbd1931.out‚0 cpu‚14.365122340225293 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.drbd1584.in‚0 proc.memory.rssMB‚ 11253.9375 proc.cpu.threads‚29$ disk./.usageGB‚587.7907791137695 disk.drbd1965.in‚0 disk.drbd1552.in‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 13.51171875 disk.rbd0.out‚0 disk.drbd1510.in‚0 disk.drbd1552.out‚0 disk.sda.out‚ 13.51171875 disk.drbd1817.in‚0 disk.drbd2041.in‚0 network.recv‚107998 disk.drbd1285.out‚0$ memory_percent‚41.22008681297302 disk.drbd1673.out‚0 disk.rbd1.out‚ 1.6630859375 disk.drbd1584.out‚0 disk.drbd1510.out‚0 disk.sda.in‚ 0.00390625 disk.drbd1828.in‚0 disk.dm-0.out‚530.7763671875 disk.drbd2078.in‚0 disk.drbd1828.out‚0v‹ÅÇŽµ‚(:ƒ žÅÓÜþ’ƒ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚94.65' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚31.55 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚07OöÎM¶‚(:  žÅÓÀÛƒ) proc.memory.percent‚17.17214584350586 network.sent‚938075 network.recv‚110673 proc.cpu.threads‚29 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.rbd1.out‚ 1.6953125 proc.memory.rssMB‚ 11253.9375 disk.sda.out‚ 13.9296875 disk.drbd1289.in‚0 disk.drbd1289.out‚0$ disk.nvme0n1.out‚531.36572265625$ memory_percent‚41.22009873390198 disk.drbd1552.in‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0) disk./.usagePercent‚32.87234126435815 disk.drbd1552.out‚0 disk.drbd1931.out‚0 disk.drbd1510.in‚0 disk.drbd2118.in‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 13.9296875 disk.drbd1673.out‚0* proc.memory.availableMB‚38521.99609375 disk.drbd1584.out‚0 disk.sda.in‚ 0.00390625 disk.drbd1550.in‚0 disk.sda1.out‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.dm-0.in‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0 cpu‚14.342561772708883 disk.drbd1828.in‚0 disk.drbd1226.out‚0$ disk./.usageGB‚587.7908477783203! disk.dm-0.out‚531.36572265625 disk.drbd1510.out‚0 disk.sda2.in‚ 0.00390625 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd2111.in‚0 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.rbd0.in‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0ÏŸ±€0·+ _wandbruntime‚435  _runtime‚435‡+Ÿ¿ _step‚76 ' lr-AdamW/pg1‚2.5746414298820792e-05 ' lr-AdamW/pg2‚2.5746414298820792e-05 trainer/global_step‚999 % _timestamp‚1.7858106904817998e+09 _runtime‚ 435.67174683]@Øɹà ' lr-AdamW/pg1‚2.5746414298820792e-05 ' lr-AdamW/pg2‚2.5746414298820792e-05 trainer/global_step‚999 % _timestamp‚1.7858106904817998e+09 _runtime‚ 435.67174683 _step‚76LÕèwj0º+ _wandbruntime‚435  _runtime‚435Ì(3ü¨'»¢O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚17212941134848 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3939395 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10566345 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 879519060480 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10556689 _runtime‚ 435.676647713 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚17171305688064 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6809594 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚16744734036832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6647579 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚18050912178176 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚17212966038528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚18050807709184 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚17171305688064 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6647577 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6809857 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚17622185986536 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚18116491185152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 877366146580 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚17171288648704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 $ _timestamp‚1.785810690482735e+09 _step‚77 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3939397 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 903687015936 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚261 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10556689 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6647577 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17204266 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚18050912178176 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 879519060480 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17213924 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚263 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 877451949704 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 879606490112 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚16744716997472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚18116653054464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2870460 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚999 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚18050807709184 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2870199 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚17622083144052 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2870460 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚999 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚18050807709184 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2870199 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚17622083144052 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚16744734036832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6647579 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚18050912178176 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚17212966038528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚18050807709184 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚17171305688064 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6647577 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6809857 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚17622185986536 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚18116491185152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 877366146580 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚17171288648704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 $ _timestamp‚1.785810690482735e+09 _step‚77 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3939397 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 903687015936 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚261 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10556689 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6647577 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚538 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17204266 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚18050912178176 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 879519060480 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17213924 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚263 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 877451949704 ]LDevice)y#StatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 879606490112 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚16744716997472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚18116653054464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200M¼ 6•0½+ _wandbruntime‚436  _runtime‚436œÉJ׳&¾­M [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 880486116864 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6654222 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚16761474209152 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 880398680064 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10576882 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6654222 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚18068854091776 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10567212 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17231104 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 878329423038 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚17188455411712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚18068958567936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚16761457169792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚18068958567936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚18134604741120 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858106908453555e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚17188472451072 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 880486116864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17231104 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2873279 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6816613 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 904589253632 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3943332 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2873029 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10576882 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 904456489472 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚18134762408960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚17188455411712 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6654220 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚17639803632190 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17221432 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6816361 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3943334 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 880398680064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10567212 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚17230148251648 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 878243619802=;ê·&¿±M E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3943334 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 880398680064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10567212 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚17230148251648 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 878243619802 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 880486116864 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6654222 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚16761474209152 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 880398680064 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10576882 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚538 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6654222 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚18068854091776 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10567212 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17231104 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 878329423038 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚17188455411712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚18068958567936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚16761457169792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚18068958567936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚18134604741120 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858106908453555e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚17188472451072 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 880486116864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17231104 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2873279 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6816613 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 904589253632 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3943332 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2873029 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10576882 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 904456489472 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚18134762408960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚17188455411712 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6654220 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚528 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚17639803632190 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17221432 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6816361 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130NÍ!˜0À+ _wandbruntime‚436  _runtime‚436+Ú(þÁø _step‚79 /loss/FRt2/assignment_loss‚5.441198377609253 % _timestamp‚1.7858106908466082e+09 /loss/FRt1/assignment_loss‚5.784183466434479 /loss/FBt2/detection_loss‚6.5591976621309795 trainer/global_step‚999 _runtime‚ 436.036396338 1loss/SRqqt1/assignment_loss‚2.325426566414535 2loss/SRqqt2/assignment_loss‚2.7424926344229252 0loss/FBt1/assignment_loss‚0.5418842553504204 1loss/FBt2/assignment_loss‚0.11363668163151791 .loss/FRt2/detection_loss‚3.166037267446518 .loss/FBt1/detection_loss‚17.60131213539525 %loss/total_loss‚52.50671833138084 /loss/FRt1/detection_loss‚3.9660091402009128 1loss/SRqqt1/detection_loss‚1.1361428813562497 1loss/SRqqt2/detection_loss‚3.1291972629877987 epoch‚0•I9Âü /loss/FBt2/detection_loss‚6.5591976621309795 trainer/global_step‚999 _runtime‚ 436.036396338 1loss/SRqqt1/assignment_loss‚2.325426566414535 2loss/SRqqt2/assignment_loss‚2.7424926344229252 0loss/FBt1/assignment_loss‚0.5418842553504204 1loss/FBt2/assignment_loss‚0.11363668163151791 .loss/FRt2/detection_loss‚3.166037267446518 .loss/FBt1/detection_loss‚17.60131213539525 %loss/total_loss‚52.50671833138084 /loss/FRt1/detection_loss‚3.9660091402009128 1loss/SRqqt1/detection_loss‚1.1361428813562497 1loss/SRqqt2/detection_loss‚3.1291972629877987 epoch‚0 _step‚79 /loss/FRt2/assignment_loss‚5.441198377609253 % _timestamp‚1.7858106908466082e+09 /loss/FRt1/assignment_loss‚5.784183466434479O¾jB>œÃ‚(:‘ žÅÓÀ²ƒ gpu.0.gpu‚35.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚120.184' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.06133333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ü´\ÄKÄ‚(:À  žÅÓ‚×èƒ disk.drbd2111.out‚0 disk.drbd2118.in‚0 disk.drbd1673.out‚0$ disk.nvme0n1.out‚531.69384765625 proc.cpu.threads‚29 disk.drbd1828.in‚0 proc.memory.rssMB‚ 11253.9375) proc.memory.percent‚17.17214584350586 disk.drbd1289.in‚0 disk.nvme0n1.in‚0 disk.sda.out‚ 14.484375% memory_percent‚41.220176219940186$ disk./.usageGB‚587.7908477783203 disk.sda1.in‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.drbd2111.in‚0 disk.drbd1289.out‚0 disk.sda2.out‚ 14.484375) disk./.usagePercent‚32.87234126435815 disk.drbd1550.in‚0 disk.drbd1285.in‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd1931.out‚0 disk.drbd2078.in‚0! disk.dm-0.out‚531.69384765625 disk.drbd1552.in‚0 disk.rbd1.out‚ 1.6953125 disk.drbd1356.in‚0 disk.drbd1285.out‚0 disk.drbd2118.out‚0 disk.drbd1938.out‚0) proc.memory.availableMB‚ 38521.9453125 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.sda.in‚ 0.00390625 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.dm-0.in‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.00390625 network.sent‚941264 network.recv‚111772 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd1552.out‚0 disk.drbd1584.in‚0 disk.drbd1965.in‚0 disk.drbd1510.in‚0 disk.drbd1226.in‚0 cpu‚14.321576990479063.;]0Å+ _wandbruntime‚456  _runtime‚456ö ±LÇÆÁ _step‚80 ' lr-AdamW/pg1‚2.7035023623086093e-05 ' lr-AdamW/pg2‚2.7035023623086093e-05 trainer/global_step‚1049 % _timestamp‚1.7858107117453885e+09 _runtime‚ 456.935337034,D£ËÇÅ ' lr-AdamW/pg2‚2.7035023623086093e-05 trainer/global_step‚1049 % _timestamp‚1.7858107117453885e+09 _runtime‚ 456.935337034 _step‚80 ' lr-AdamW/pg1‚2.7035023623086093e-05P4L[·0È+ _wandbruntime‚456  _runtime‚456sUå˜êɤO =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11094347 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚19022005590528 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3013197 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6979727 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11084691 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚18073296974848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 % _timestamp‚1.7858107117463732e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 923515195904 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 921340413212 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4136245 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚17581742652832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚17581725613472 _runtime‚ 456.939677237 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚18029643838464 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚281 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 948708615680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6979727 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 923515195904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚531 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚18953141995008 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 923602625536 _step‚81 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚19022163265536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7149725 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 trainer/global_step‚1049 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18074076 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11094347 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚18502980223560 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚18953141995008 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚18029626799104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 921340413212 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4136245 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚17581742652832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚17581725613472 _runtime‚ 456.939677237 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚18029643838464 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚281 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 948708615680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6979727 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 923515195904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚531 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚18953141995008 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 923602625536 _step‚81 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚19022163265536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7149725 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 trainer/global_step‚1049 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18074076 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11094347 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚18502980223560 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚18953141995008 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚18029626799104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18081584 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 457.304184009 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6986370 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 922132083310 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4140184 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3016026 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7156208 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 922217886546 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _step‚82 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11104884 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚541 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚18520597869102 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4140182 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18090504091648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18091256 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚18046793562112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18091256 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7156462 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18046810601472 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚18971188377600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6986372øŸêõ¸&ͲM VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚18971292853760 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 924394815488 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 trainer/global_step‚1049 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚19040271587840 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚19040113920000 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 949609828352 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 924482252288 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚18090528995328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚18520700711698 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11095214 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6986370 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18091256 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7156462 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18046810601472 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚18971188377600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6986372 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6986372 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3016278 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18081584 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 457.304184009 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6986370 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 922132083310 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4140184 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3016026 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7156208 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 922217886546 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _step‚82 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11104884 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚541 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚18520597869102 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4140182 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18090504091648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18091256 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚18046793562112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4333097 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚18418734229472 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚19855476280832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291oEóy­'اO XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚18887964949504 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4333097 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚18418734229472 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚19855476280832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 967511331328 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚19383877303068 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚19855580749824 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚18887981988864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11612693 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚19855476280832 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4333095 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 993940961792 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 965143073596 _step‚85UqßÔ0Ù+ _wandbruntime‚479  _runtime‚479 †Òø"Ú®M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚541 _step‚86 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚18435474401792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 968478387712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚19401597791206 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11632886 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4337032 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3160619 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 % _timestamp‚1.7858107342565982e+09 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7497653 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11632886 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚19873627139584 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18905148751872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚19401494948610 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚19873522663424 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18951408 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚1099 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚19945703091200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18950859931648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18941736 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11623216 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚18905148751872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚18905131712512 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7318522 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 968390950912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11623216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚18435491441152 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 994843159552 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 966106350054 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7318520 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 968390950912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18941736 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7318520 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7497407 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚18950884835328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes "q¼.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 479.446917301 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18951408 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0r2Çž¸&Û²M E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚1099 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚19945703091200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18950859931648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18941736 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11623216 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚18905148751872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚18905131712512 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7318522 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 968390950912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11623216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚18435491441152 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 994843159552 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 966106350054 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7318520 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 968390950912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18941736 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7318520 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7497407 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚18950884835328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 479.446917301 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18951408 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚541 _step‚86 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚18435474401792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 968478387712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚19401597791206 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11632886 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4337032 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3160619 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 % _timestamp‚1.7858107342565982e+09 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7497653 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11632886 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚19873627139584 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18905148751872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚19401494948610 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚19873522663424 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18951408 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130V¥Q3Ã0Ü+ _wandbruntime‚479  _runtime‚479ÿ)?fþÝø %loss/total_loss‚56.82401671864264 epoch‚0 .loss/FBt1/detection_loss‚20.80989002576098 % _timestamp‚1.7858107342578986e+09 _runtime‚ 479.447699826 trainer/global_step‚1099 _step‚87 0loss/FRt1/assignment_loss‚5.4084697484970095 .loss/FBt2/detection_loss‚7.200340437664481 /loss/FRt2/assignment_loss‚5.071536295192757 2loss/SRqqt1/assignment_loss‚2.2413075478180593 1loss/SRqqt2/assignment_loss‚2.668811234144064 0loss/FBt1/assignment_loss‚0.5795691821258515 .loss/FRt1/detection_loss‚4.015298271551728 /loss/FRt2/detection_loss‚3.3512490437817326 0loss/SRqqt2/detection_loss‚3.961888850881503 1loss/FBt2/assignment_loss‚0.15275493634140325 1loss/SRqqt1/detection_loss‚1.3629011448830683‹ß½ Þü .loss/FBt2/detection_loss‚7.200340437664481 /loss/FRt2/assignment_loss‚5.071536295192757 2loss/SRqqt1/assignment_loss‚2.2413075478180593 1loss/SRqqt2/assignment_loss‚2.668811234144064 0loss/FBt1/assignment_loss‚0.5795691821258515 .loss/FRt1/detection_loss‚4.015298271551728 /loss/FRt2/detection_loss‚3.3512490437817326 0loss/SRqqt2/detection_loss‚3.961888850881503 1loss/FBt2/assignment_loss‚0.15275493634140325 1loss/SRqqt1/detection_loss‚1.3629011448830683 %loss/total_loss‚56.82401671864264 epoch‚0 .loss/FBt1/detection_loss‚20.80989002576098 % _timestamp‚1.7858107342578986e+09 _runtime‚ 479.447699826 trainer/global_step‚1099 _step‚87 0loss/FRt1/assignment_loss‚5.4084697484970095Ws%û›ß‚(: ®žÅÓëù¡ƒ gpu.0.gpu‚32.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚97.822' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.60733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦µµbKà‚(:À  ®žÅÓÀŒÆƒ disk.sda1.out‚0 disk.drbd1938.in‚0 disk.drbd1356.in‚0 network.sent‚ 1.031238e+06 disk.sda2.out‚15.40625 disk.drbd1584.in‚0 disk.drbd1817.in‚0 disk.drbd2078.in‚0 disk.drbd2078.out‚0 disk.sda1.in‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0 proc.cpu.threads‚29 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0) proc.memory.availableMB‚ 38521.5546875 disk.drbd1817.out‚0 disk.drbd1828.out‚0! disk.dm-0.out‚532.16552734375 disk.drbd2111.in‚0 disk.drbd1550.in‚0 disk.drbd1289.in‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0) disk./.usagePercent‚32.87235641134485 disk.drbd1510.in‚0 disk.drbd1552.out‚0 disk.nvme0n1.in‚0 disk.dm-0.in‚0 disk.drbd1510.out‚0 disk.sda.in‚ 0.00390625 disk.drbd1226.out‚0 disk.drbd2118.out‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1931.in‚0 disk.drbd1550.out‚0 network.recv‚116221 disk.drbd1285.out‚0 disk.drbd2118.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚1.9375 disk.sda.out‚15.40625 cpu‚14.283571907539528 disk.rbd0.in‚0 disk.drbd1938.out‚0$ disk./.usageGB‚587.7911186218262 proc.memory.rssMB‚ 11253.9375$ memory_percent‚41.22077226638794 disk.drbd2111.out‚0 disk.drbd1289.out‚0) proc.memory.percent‚17.17214584350586 disk.rbd0.out‚0 disk.sda2.in‚ 0.00390625$ disk.nvme0n1.out‚532.16552734375 disk.drbd2041.in‚0 disk.drbd1965.in‚0áÜ®Á›á‚(: ½žÅӧ̡ƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚122.77' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.92333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0OZ»Vâ‚(:Ë  ½žÅÓ¤¡ìƒ network.recv‚118919 disk.drbd2111.out‚0 disk.drbd1938.in‚0 disk.drbd1226.in‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1584.out‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.dm-0.in‚0$ disk./.usageGB‚587.7910957336426$ disk.nvme0n1.out‚532.32177734375 disk.drbd1550.out‚0! disk.dm-0.out‚532.32177734375 disk.drbd2111.in‚0$ memory_percent‚41.22081995010376 disk.drbd1817.in‚0 disk.sda2.out‚ 15.75390625 disk.rbd1.out‚ 1.9697265625) proc.memory.percent‚17.17214584350586 proc.cpu.threads‚29 disk.drbd1817.out‚0 disk.drbd1828.in‚0) proc.memory.availableMB‚ 38521.5234375 cpu‚14.264958310122276 network.sent‚ 1.076339e+06 disk.drbd1965.out‚0 disk.drbd1828.out‚0 disk.drbd2041.out‚0 disk.sda.out‚ 15.75390625 disk.drbd1550.in‚0 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.rbd0.out‚0 disk.drbd1931.out‚0 disk.sda.in‚ 0.00390625 disk.drbd1584.in‚0 disk.drbd1510.in‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.drbd1289.out‚0 disk.drbd1938.out‚0 disk.sda1.in‚0 disk.drbd1552.out‚0( disk./.usagePercent‚32.8723551313178 disk.drbd1673.in‚0 disk.drbd1356.out‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1673.out‚0 disk.rbd0.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.00390625 disk.nvme0n1.in‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1510.out‚0 proc.memory.rssMB‚ 11253.9375Î7ë0ã+ _wandbruntime‚500  _runtime‚500„ŠjÇäÁ ' lr-AdamW/pg2‚2.9612242271616706e-05 trainer/global_step‚1149 % _timestamp‚1.7858107554859922e+09 _runtime‚ 500.675933325 _step‚88 ' lr-AdamW/pg1‚2.9612242271616706e-05ðœXËåÅ _step‚88 ' lr-AdamW/pg1‚2.9612242271616706e-05 ' lr-AdamW/pg2‚2.9612242271616706e-05 trainer/global_step‚1149 % _timestamp‚1.7858107554859922e+09 _runtime‚ 500.675933325X~0æ+ _wandbruntime‚500  _runtime‚500$)µ±'ç«O IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚259 _runtime‚ 500.680638151 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚20757810566656 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚20757810566656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚20833113201664 trainer/global_step‚1149 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4529945 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7644027 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1011594896384 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3301679 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚20264774382576 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1009031537104 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19794380 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚20264877225060 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1011507466752 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚19746320139264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7644029 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚20757915035648 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7831365 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4529947 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _step‚89 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚19746303099904 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚19746320139264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚19746303099904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7644027 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚20757915035648 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚541 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1039104546816 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12150351 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12140695 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1009117340228 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1011507466752 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656€A;Üyè¯O dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚19794033558528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚261 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚19794008654848 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1011594896384 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12150351 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7831626 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1039243609600 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 $ _timestamp‚1.785810755486902e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚19784722 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12140695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚531 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚19255742845472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚20757915035648 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7831365 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4529947 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _step‚89 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚19746303099904 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚19746320139264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚19746303099904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7644027 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚20757915035648 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 1ðç§b<04468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚541 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1039104546816 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12150351 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12140695 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1009117340228 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1011507466752 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚259 _runtime‚ 500.680638151 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚20757810566656 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚20757810566656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚20833113201664 trainer/global_step‚1149 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4529945 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7644027 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1011594896384 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3301679 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚20264774382576 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1009031537104 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19794380 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚20264877225060 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1011507466752 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚19746320139264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7644029 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1009994813562 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7838210 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1040019300864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚20775961425408 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1040143676416 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12151218 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚19272500057152 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚19763469862912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1012474523136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚20775961425408 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7650670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚19811215771648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3304328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1009909010326 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚19811240675328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785810755843958e+09 _step‚90 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚19763469862912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19811560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚ 501.034255599 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7650670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1012387086336 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚19763486902272 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚20851235072512 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 trainer/global_step‚1149 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7650672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12160888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4533884 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12151218 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12160888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚19763486902272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1012387086336 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7838473 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚20282494870714 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19811560 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7650672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0³ŒYÁ¿&ë¹M ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚19763486902272 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚20851235072512 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 trainer/global_step‚1149 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7650672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12160888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4533884 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12151218 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12160888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚19763486902272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1012387086336 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7838473 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚20282494870714 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19811560 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7650672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1009994813562 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7838210 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1040019300864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚20775961425408 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1040143676416 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12151218 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚19272500057152 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚19763469862912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1012474523136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚20775961425408 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7650670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚19811215771648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3304328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1009909010326 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚19811240675328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785810755843958e+09 _step‚90 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚19763469862912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19811560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚ 501.034255599 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7650670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1012387086336 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121ZÐæ:—0ì+ _wandbruntime‚501  _runtime‚501hÄCýí÷ 2loss/SRqqt2/assignment_loss‚2.6790517568588257 /loss/FRt2/detection_loss‚3.8571611433511688 %loss/total_loss‚60.86670864482021 trainer/global_step‚1149 1loss/FBt2/assignment_loss‚0.22770566460773503 0loss/SRqqt2/detection_loss‚4.109246346155802 epoch‚0 $ _timestamp‚1.785810755845186e+09 _runtime‚ 501.034974901 .loss/FRt1/detection_loss‚4.758453159634747 1loss/SRqqt1/detection_loss‚2.2516045799902518 /loss/FBt1/detection_loss‚21.041778256651014 -loss/FBt2/detection_loss‚9.24152385742151 /loss/FRt1/assignment_loss‚5.284157397142097 /loss/FRt2/assignment_loss‚4.518245762302762 2loss/SRqqt1/assignment_loss‚2.2842547081886453 0loss/FBt1/assignment_loss‚0.6135260125156492 _step‚91([îû _runtime‚ 501.034974901 .loss/FRt1/detection_loss‚4.758453159634747 1loss/SRqqt1/detection_loss‚2.2516045799902518 /loss/FBt1/detection_loss‚21.041778256651014 -loss/FBt2/detection_loss‚9.24152385742151 /loss/FRt1/assignment_loss‚5.284157397142097 /loss/FRt2/assignment_loss‚4.518245762302762 2loss/SRqqt1/assignment_loss‚2.2842547081886453 0loss/FBt1/assignment_loss‚0.6135260125156492 _step‚91 2loss/SRqqt2/assignment_loss‚2.6790517568588257 /loss/FRt2/detection_loss‚3.8571611433511688 %loss/total_loss‚60.86670864482021 trainer/global_step‚1149 1loss/FBt2/assignment_loss‚0.22770566460773503 0loss/SRqqt2/detection_loss‚4.109246346155802 epoch‚0 $ _timestamp‚1.785810755845186e+09[¦¨©Ó›ï‚(: ÌžÅӎ憃 gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚126.88' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.29333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ðÕÝ¿Cð‚(:¸  ÌžÅÓ§¾Ñƒ disk.nvme0n1.in‚0 disk.drbd2041.in‚0 disk.drbd1289.out‚0 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1938.in‚0 disk.sda.in‚ 0.00390625 disk.rbd0.in‚0 disk.dm-0.in‚0 disk.drbd1673.out‚0 disk.drbd1584.in‚0 disk.drbd1965.out‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.drbd2041.out‚0 network.recv‚120233 disk.drbd1828.in‚0! disk.dm-0.out‚532.75048828125 disk.drbd1938.out‚0 disk.sda2.in‚ 0.00390625 disk.drbd1931.out‚0) proc.memory.percent‚17.17214584350586 disk.drbd1285.in‚0' proc.memory.availableMB‚ 38521.71875 disk.sda1.out‚0 disk.drbd1550.out‚0 network.sent‚ 1.079618e+06 disk.drbd1584.out‚0 disk.sda.out‚16 disk.drbd2118.out‚0 disk.drbd1550.in‚0 disk.drbd1356.in‚0 disk.drbd1931.in‚0 disk.drbd1552.in‚0 disk.drbd1817.out‚0$ disk.nvme0n1.out‚532.75048828125 disk.drbd1356.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.drbd1226.out‚0 disk.sda1.in‚0 disk.rbd1.out‚ 1.9697265625$ memory_percent‚41.22052192687988* disk./.usagePercent‚32.872358544723255 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.sda2.out‚16 proc.memory.rssMB‚ 11253.9375 proc.cpu.threads‚29 disk.drbd2078.out‚0 disk.drbd1510.in‚0 disk.rbd1.in‚0 cpu‚14.24930899410518$ disk./.usageGB‚587.7911567687988Âû~ž0ñ+ _wandbruntime‚522  _runtime‚522+€öKÅò¿ & lr-AdamW/pg1‚3.090085159588201e-05 & lr-AdamW/pg2‚3.090085159588201e-05 trainer/global_step‚1199 % _timestamp‚1.7858107768226726e+09 _runtime‚ 522.012631264 _step‚92‘åšsÉóà trainer/global_step‚1199 % _timestamp‚1.7858107768226726e+09 _runtime‚ 522.012631264 _step‚92 & lr-AdamW/pg1‚3.090085159588201e-05 & lr-AdamW/pg2‚3.090085159588201e-05\rÛ’‰0ô+ _wandbruntime‚522  _runtime‚522ؤ÷–¾õ¬O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1055591031808 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1055503602176 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚21738683292672 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1084455763456 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚541 _step‚93 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12668697 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚20092751461472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1055503602176 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚21738845161984 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚20654364494848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚531 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1052920000612 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚20604641250304 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚21145671462084 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8172728 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20644874 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚20654389398528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858107768236516e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WKDeviceStat×fvNôsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4726797 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1084318797824 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7976177 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12678353 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚21660249321472 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚21660144852480 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12668697 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7976179 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚ 522.013871712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚20604658289664 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚20654532 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7976177 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚21660144852480hgtN¶'ö°O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚20604658289664 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚20654532 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7976177 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚21660144852480 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1055591031808 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1055503602176 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚21738683292672 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1084455763456 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚541 _step‚93 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12668697 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚20092751461472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1055503602176 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚21738845161984 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚20654364494848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚531 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1052920000612 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚20604641250304 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚21145671462084 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8172728 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20644874 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚20654389398528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858107768236516e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4726797 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1084318797824 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7976177 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12678353 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚21660249321472 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚21660144852480 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12668697 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7976179 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚ 522.013871712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856]ÝÆ20÷+ _wandbruntime‚522  _runtime‚522†~¼&ø¶M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12688890 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1056470658560 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1085215635968 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚21163391950222 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7982822 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 143249920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚21163289107626 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚20671596515328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7982820 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1053797473834 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7982820 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8179770 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12679220 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1085358885888 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚21678295711232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚21756955401216 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚20662040 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚20621825052672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚20621808013312 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8179508 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1056383221760 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3448776 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚20109508673152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚20671712 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚21678295711232 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20671712 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7982822 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200GùºM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7982820 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1053797473834 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7982820 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8179770 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12679220 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1085358885888 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚21678295711232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚21756955401216 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚20662040 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚20621825052672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚20621808013312 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8179508 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1056383221760 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3448776 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚20109508673152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚20671712 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚21678295711232 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20671712 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7982822 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMžÝÊ y onitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12688890 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1056470658560 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1085215635968 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚21163391950222 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7982822 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 143249920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚21163289107626 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚20671596515328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392^DZßØ0ú+ _wandbruntime‚522  _runtime‚522ÈM]Áûûõ .loss/FRt1/detection_loss‚5.034113530910784 -loss/FRt2/detection_loss‚3.19334853542786 .loss/FBt2/detection_loss‚5.040195530752085 _step‚95 0loss/FRt2/assignment_loss‚4.1539112937216665 0loss/SRqqt1/assignment_loss‚2.23355117933018 1loss/SRqqt2/assignment_loss‚2.647881580304496 1loss/FBt2/assignment_loss‚0.10364252516592387 1loss/SRqqt2/detection_loss‚3.9175006385090985 .loss/FBt1/detection_loss‚37.22458822776874 %loss/total_loss‚70.61066084445837 trainer/global_step‚1199 1loss/SRqqt1/detection_loss‚1.3702905371016147 _runtime‚ 522.378788586 epoch‚0 % _timestamp‚1.7858107771888123e+09 .loss/FRt1/assignment_loss‚5.34597179197496 1loss/FBt1/assignment_loss‚0.34566547349095345 8Òÿüù 0loss/FRt2/assignment_loss‚4.1539112937216665 0loss/SRqqt1/assignment_loss‚2.23355117933018 1loss/SRqqt2/assignment_loss‚2.647881580304496 1loss/FBt2/assignment_loss‚0.10364252516592387 1loss/SRqqt2/detection_loss‚3.9175006385090985 .loss/FBt1/detection_loss‚37.22458822776874 %loss/total_loss‚70.61066084445837 trainer/global_step‚1199 1loss/SRqqt1/detection_loss‚1.3702905371016147 _runtime‚ 522.378788586 epoch‚0 % _timestamp‚1.7858107771888123e+09 .loss/FRt1/assignment_loss‚5.34597179197496 1loss/FBt1/assignment_loss‚0.34566547349095345 .loss/FRt1/detection_loss‚5.034113530910784 -loss/FRt2/detection_loss‚3.19334853542786 .loss/FBt2/detection_loss‚5.040195530752085 _step‚95_W!îNý‚(:à  ÛžÅÓèáÞƒ disk.drbd1356.in‚0 disk.rbd0.out‚0 disk.sda.out‚16.34375 disk.drbd1285.out‚0 disk.drbd2118.out‚0 proc.memory.rssMB‚ 11253.9375) proc.memory.percent‚17.17214584350586 disk.rbd1.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0) disk./.usagePercent‚32.87235641134485$ disk./.usageGB‚587.7911186218262 disk.sda2.in‚ 0.00390625 disk.drbd1584.out‚0 disk.drbd2111.out‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.drbd1965.out‚0 disk.drbd1828.in‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.dm-0.in‚0 disk.drbd1931.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.drbd1552.in‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.sda2.out‚16.34375 network.sent‚ 1.124847e+06 network.recv‚122804 disk.rbd1.out‚ 2.1259765625 disk.drbd1510.in‚0 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0% proc.memory.availableMB‚ 38521.625% memory_percent‚41.220664978027344 disk.nvme0n1.in‚0 disk.drbd1356.out‚0 disk.sda.in‚ 0.00390625 disk.drbd2078.out‚0 disk.drbd1289.in‚0 disk.drbd2111.in‚0 disk.drbd1552.out‚0 proc.cpu.threads‚29$ disk.nvme0n1.out‚532.92626953125 disk.drbd2041.out‚0! disk.dm-0.out‚532.92626953125 disk.sda1.in‚0 disk.drbd1673.out‚0 cpu‚14.232528873441355ìÉ ¨œþ‚(:‘ ÛžÅÓ”Ðòƒ gpu.0.gpu‚34.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚129.808' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.26933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0j€$‘ÿ‚(:† êžÅÓ÷Àú‚ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚108.642' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚36.214 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¯œS€‚(:È  êžÅÓեă disk.dm-0.in‚0 disk.sda1.in‚0 disk.drbd1584.out‚0 disk.drbd1226.in‚0 disk.dm-0.out‚533.4462890625 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.rbd1.in‚0 disk.drbd1965.out‚0 disk.drbd1828.in‚0 disk.drbd1938.out‚0# disk.nvme0n1.out‚533.4462890625$ disk./.usageGB‚587.7903213500977 disk.rbd0.out‚0 disk.sda2.out‚ 16.7890625 disk.drbd2041.out‚0* proc.memory.availableMB‚38520.72265625 proc.memory.rssMB‚ 11253.9375 disk.drbd1510.in‚0 disk.sda1.out‚0 disk.drbd1673.in‚0 cpu‚14.2182570452834) disk./.usagePercent‚32.87231182373611 disk.drbd1226.out‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.sda.in‚ 0.00390625 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1356.out‚0 network.recv‚126407 disk.drbd1828.out‚0 disk.drbd1552.out‚0 disk.drbd1356.in‚0 disk.drbd1931.in‚0 disk.rbd0.in‚0 proc.cpu.threads‚29 disk.drbd2078.in‚0 disk.drbd1550.out‚0 disk.drbd1289.out‚0 disk.drbd1550.in‚0 disk.sda2.in‚ 0.00390625) proc.memory.percent‚17.17214584350586 disk.drbd1965.in‚0 disk.rbd1.out‚ 2.1259765625 disk.drbd1931.out‚0 disk.drbd2078.out‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0% memory_percent‚41.222041845321655 network.sent‚ 1.170325e+06 disk.drbd1285.out‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚0 disk.sda.out‚ 16.7890625§µÓt0+ _wandbruntime‚543  _runtime‚543vžtÄ‚¾ & lr-AdamW/pg1‚3.218946092014731e-05 & lr-AdamW/pg2‚3.218946092014731e-05 trainer/global_step‚1249 $ _timestamp‚1.785810798593676e+09 _runtime‚ 543.785621536 _step‚96ûôîaȃ _step‚96 & lr-AdamW/pg1‚3.218946092014731e-05 & lr-AdamW/pg2‚3.218946092014731e-05 trainer/global_step‚1249 $ _timestamp‚1.785810798593676e+09 _runtime‚ 543.785621536`•?c0„+ _wandbruntime‚543  _runtime‚543¤ûYq²'…¬O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1099499737600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13196699 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1099587167232 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8512918 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4923647 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3589015 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚21462996440064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚22026671384076 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 trainer/global_step‚1249 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1096808464120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚22562583607296 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚541 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1129584648192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1129451876864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚22562479138304 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8308327 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚21514720334848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8512660 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚21462996440064 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚21514745238528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22026568541592 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚20929777116832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚22562583607296 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚22562479138304 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚22644172211712 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13206355 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1099499737600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21514684 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8308329 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚531 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8308329 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0ß#÷¶'†°O GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13206355 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1099499737600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21514684 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8308329 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚531 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8308329 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1099499737600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13196699 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1099587167232 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8512918 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4923647 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3589015 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚21462996440064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚22026671384076 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 trainer/global_step‚1249 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1096808464120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚22562583607296 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚541 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1129584648192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1129451876864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚22562479138304 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8308327 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚21514720334848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8512660 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚21462996440064 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚21514745238528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22026568541592 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚20929777116832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚22562583607296 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚22562479138304 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚22644172211712 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2aP–î 0‡+ _wandbruntime‚544  _runtime‚5442.› ˆ´M JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚20946517289152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚22580629997056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1100379357184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21531864 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3592095 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚21480163203072 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3591845 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8314972 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚21531927451648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8519427 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚21480146163712 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1097771740578 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚21522192 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4927584 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚21480146163712 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1100466793984 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚ 544.135841935 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21531864 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4927582 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚22662282966016 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8314970 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚21480163203072 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1130486181376 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚1249 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13216892 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚22580629997056 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚22580525520896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚21531952355328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚22044289029730 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1100466793984 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _step‚98 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1097685937342 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13207222 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8314972 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8519427 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚21480146163712 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1097771740578 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚21522192 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4927584 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚21480146163712 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1100466793984 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚ 544.135841935 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21531864 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4927582 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚22662282966016 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8314970 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚21480163203072 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1130486181376 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚1249 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13216892 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚22580629997056 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚22580525520896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚21531952355328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚22044289029730 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1100466793984 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _step‚98 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1097685937342 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13207222 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8314972 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚21480163203072 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3591845 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136bJá(æ0Š+ _wandbruntime‚544  _runtime‚544ÝÊ–Áþ‹ø .loss/FRt1/assignment_loss‚5.21788926558061 /loss/FRt2/detection_loss‚3.3737576686844384 _step‚99 2loss/SRqqt1/assignment_loss‚2.3554812415132247 1loss/FBt2/assignment_loss‚0.23760671774300957 /loss/FBt2/detection_loss‚5.0566447968555845 epoch‚0 % _timestamp‚1.7858107989468067e+09 /loss/FRt2/assignment_loss‚4.133771972557933 0loss/FBt1/assignment_loss‚0.4148785786727737 .loss/FBt1/detection_loss‚16.96649397654753 &loss/total_loss‚51.104082842948415 trainer/global_step‚1249 2loss/SRqqt2/assignment_loss‚2.6609678732884396 .loss/FRt1/detection_loss‚4.854633892801675 1loss/SRqqt1/detection_loss‚1.8573636237895046 /loss/SRqqt2/detection_loss‚3.97459323491369 _runtime‚ 544.136606216a–ØþŒü /loss/FRt2/assignment_loss‚4.133771972557933 0loss/FBt1/assignment_loss‚0.4148785786727737 .loss/FBt1/detection_loss‚16.96649397654753 &loss/total_loss‚51.104082842948415 trainer/global_step‚1249 2loss/SRqqt2/assignment_loss‚2.6609678732884396 .loss/FRt1/detection_loss‚4.854633892801675 1loss/SRqqt1/detection_loss‚1.8573636237895046 /loss/SRqqt2/detection_loss‚3.97459323491369 _runtime‚ 544.136606216 .loss/FRt1/assignment_loss‚5.21788926558061 /loss/FRt2/detection_loss‚3.3737576686844384 _step‚99 2loss/SRqqt1/assignment_loss‚2.3554812415132247 1loss/FBt2/assignment_loss‚0.23760671774300957 /loss/FBt2/detection_loss‚5.0566447968555845 epoch‚0 % _timestamp‚1.7858107989468067e+09c+ò-—Y‚(:Î  ùžÅÓ̃¸ƒ disk.sda.out‚ 17.12890625 disk.drbd1226.out‚0# disk.nvme0n1.out‚533.6220703125 disk.drbd1584.in‚0 disk.sda2.out‚ 17.12890625 disk.drbd1931.out‚0 cpu‚14.204470438177642 disk.dm-0.in‚0 disk.drbd1817.in‚0 disk.drbd1285.out‚0 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd1356.out‚0$ memory_percent‚41.22136831283569# proc.memory.rssMB‚ 11253.9453125 proc.cpu.threads‚29 disk.drbd2118.out‚0 disk.drbd1226.in‚0 disk.drbd1938.out‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0* disk./.usagePercent‚32.872317583857814 disk.sda1.out‚0 disk.sda2.in‚ 0.00390625 disk.rbd0.in‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 network.recv‚127529 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0$ disk./.usageGB‚587.7904243469238 disk.drbd2078.in‚0 disk.drbd1828.in‚0 disk.dm-0.out‚533.6220703125 disk.drbd1510.in‚0 disk.rbd1.in‚0 disk.drbd1356.in‚0) proc.memory.availableMB‚ 38521.1640625 disk.drbd1289.in‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚0* proc.memory.percent‚17.172157764434814 network.sent‚ 1.173524e+06 disk.drbd1550.out‚0 disk.drbd2041.out‚0 disk.sda.in‚ 0.00390625 disk.drbd1550.in‚0 disk.drbd1289.out‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1938.in‚0 disk.drbd2041.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.drbd1673.out‚0 disk.rbd1.out‚ 2.158203125 disk.drbd2078.out‚0ž™ö°œŽ‚(:‘ ùžÅÓç–æ„ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.895' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.63166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0:eÙ©0+ _wandbruntime‚565  _runtime‚565mÑCÆÀ & lr-AdamW/pg1‚3.347807024441262e-05 & lr-AdamW/pg2‚3.347807024441262e-05 trainer/global_step‚1299 % _timestamp‚1.7858108199269705e+09 _runtime‚ 565.116960373 _step‚100™)dWÊ‘Ä _step‚100 & lr-AdamW/pg1‚3.347807024441262e-05 & lr-AdamW/pg2‚3.347807024441262e-05 trainer/global_step‚1299 % _timestamp‚1.7858108199269705e+09 _runtime‚ 565.116960373d5»:b0’+ _wandbruntime‚565  _runtime‚565åãK³'“­O 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚243 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1143583302656 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1143495873024 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚23464813424128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13724701 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚22375076174848 _step‚101 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚22321317551104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚22375101078528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 % _timestamp‚1.7858108199279974e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13734357 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1174801725952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚22321334590464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _runtime‚ 565.121673272 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22907465621100 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5120497 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚21766785732832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1174671051776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚23464917893120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3731265 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚23549747226624­, ‡”±O [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚21766768693472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8851760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1140696927628 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8640479 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚541 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_©þ‹…·"train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8640477 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚23464813424128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13724701 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚22375076174848 _step‚101 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚22321317551104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚22375101078528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 % _timestamp‚1.7858108199279974e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13734357 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1174801725952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚22321334590464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _runtime‚ 565.121673272 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22907465621100 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5120497 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚21766785732832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1174671051776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚23464917893120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3731265 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚23549747226624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚243 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1143583302656 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1143495873024 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8858563 $ _timestamp‚1.785810820293273e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22392016 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1144375492608 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3734131 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1141574400850 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚541 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚23482859806720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8647120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13744894 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚ 565.483600294 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13735224 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1144462929408 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚23482964282880 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚23568014526464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚23482964282880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1144375492608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22382344 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22382344 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22392016 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚21783508865792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚22338484314112 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8647120 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5124434 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5124432 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚102 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8647122 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8647122 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚22338501353472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚22925083266642 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚21783525905152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1141660204086 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚23567856858624 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚22338501353472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1175706331136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 trainer/global_step‚1299 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13735224 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8858563 $ _timestamp‚1.785810820293273e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22392016 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1144375492608 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3734131 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1141574400850 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚541 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚23482859806720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚531 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8647120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13744894 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚ 565.483600294 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13735224 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1144462929408 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚23482964282880 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚23568014526464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚23482964282880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1144375492608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22382344 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22382344 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22392016 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚21783508865792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚22338484314112 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8647120 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5124434 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5124432 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚102 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8647122 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8647122 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚22338501353472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚22925083266642 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚21783525905152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1141660204086 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚23567856858624 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚22338501353472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1175706331136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 trainer/global_step‚1299 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13735224 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚23179672740864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚1349 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8972629 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1219976393216 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚23235432014848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1219849913344 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚24367252178944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚23179655701504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23234988 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1184671194260 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14262359 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚23788465543092 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚105 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23225330 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14262359 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3874199 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚23179672740864 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚22603777309472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚24455281928192 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚23788362700608 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14252703 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1187579438080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚22603794348832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1187492008448 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚24455433311744 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1184585391136 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1187579438080 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚23235456918528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9191799 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 586.833646909 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14252703 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5317345 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3874452 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9191544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 $ _timestamp‚1.785810841640203e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚23179655701504 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚24367147709952 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8972627 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5317347 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8972627 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1187492008448˜ÁpÁ¶'¤°O G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3874452 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9191544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 $ _timestamp‚1.785810841640203e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚23179655701504 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚24367147709952 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8972627 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5317347 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8972627 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1187492008448 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚24367147709952 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚23179672740864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚1349 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8972629 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1219976393216 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚23235432014848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1219849913344 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚24367252178944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚23179655701504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23234988 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1184671194260 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14262359 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚23788465543092 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚105 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23225330 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14262359 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3874199 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚23179672740864 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚22603777309472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚24455281928192 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚23788362700608 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14252703 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1187579438080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚22603794348832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1187492008448 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚24455433311744 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1184585391136 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1187579438080 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚23235456918528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9191799 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 586.833646909 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14252703 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5317345 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130iDw‹Ù0¥+ _wandbruntime‚587  _runtime‚5874`̽&¦·M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚23196839503872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14263226 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚24385194092544 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9198248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8979270 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23252168 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚23242496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9198492 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚106 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚24473388111360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚23252664035328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚24385298568704 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚1349 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14272896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1188371628032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚23806083188746 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8979272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1188459064832 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1188459064832 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5321282 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 % _timestamp‚1.7858108420059354e+09 _runtime‚ 587.196219964 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚23196822464512 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1220748979712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8979270 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1185462864358 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23242496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚23196839503872 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3876966 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8979272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3877208 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5321284 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1188371628032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚24385298568704 Opïê&_train_batch_end/requested_bytes.small_pool.allocated‚ 1185548667594 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚23196839503872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14263226 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚24385194092544 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9198248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8979270 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23252168 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚23242496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9198492 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚106 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚24473388111360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚23252664035328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚24385298568704 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚1349 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14272896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1188371628032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚23806083188746 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8979272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1188459064832 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1188459064832 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5321282 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 % _timestamp‚1.7858108420059354e+09 _runtime‚ 587.196219964 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚23196822464512 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1220748979712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8979270 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1185462864358 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23242496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚23196839503872 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3876966 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8979272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3877208 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5321284 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1188371628032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚24385298568704 ODeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚1399 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1231488143872 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚25269481995776 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14780705 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14790361 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1231575573504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚24669362622600 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚23440785925472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚24037993851904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _step‚109 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚24037993851904 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9304779 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚271 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1228559657768 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚25360784572416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9304779 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1228473854644 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5514197 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1231575573504 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚23440802964832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 608.13351183 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚25269481995776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4016599 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚25269586464768 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9530796 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9304777 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5514195 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1231488143872 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24095140 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚273 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24095787854848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚24669259780116 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9304777 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚24038010891264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14790361 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9530523 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 æjÜ!²°O UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚24095812758528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚24085482 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858108629421527e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _step‚109 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚24037993851904 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9304779 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚271 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1228559657768 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚25360784572416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9304779 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1228473854644 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5514197 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1231575573504 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚23440802964832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 608.13351183 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚25269481995776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4016599 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚25269586464768 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9530796 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9304777 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5514195 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1231488143872 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24095140 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚273 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24095787854848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚24669259780116 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9304777 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚24038010891264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14790361 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9530523 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚24038010891264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚1399 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1231488143872 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚25269481995776 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14780705 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14790361 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1231575573504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚24669362622600 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚23440785925472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚24037993851904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDev“RbPÚiceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9537270 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1232367763456 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚110 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚25287528378368 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9311422 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚25379054632960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚260 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚23457543137152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14791228 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5518132 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1266034757632 trainer/global_step‚1399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24113019875328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1232455200256 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24112320 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5518134 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9537530 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858108633065674e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4019138 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚23457526097792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚25287632854528 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9311420 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚24112994971648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚24112320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14800898 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1232455200256 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1229351327866 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚24686980268254 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚ 608.496870483 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚24055177654272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4019396 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14791228 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚24055160614912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24102648 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚24055177654272 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24102648 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚24055160614912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚25287632854528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1232367763456a CÄÁ&µ»M MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4019396 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14791228 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚24055160614912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24102648 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚24055177654272 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24102648 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚24055160614912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚25287632854528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1232367763456 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9311422 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚24686877425658 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1265899896320 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14800898 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9537270 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1232367763456 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚110 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚25287528378368 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9311422 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚25379054632960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚260 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚23457543137152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14791228 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5518132 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1266034757632 trainer/global_step‚1399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24113019875328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1232455200256 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24112320 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5518134 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9537530 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858108633065674e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4019138 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚23457526097792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚25287632854528 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9311420 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚24112994971648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚24112320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14800898 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1232455200256 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1229351327866 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚24686980268254 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚ 608.496870483 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚24055177654272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160n¡(_¼0¶+ _wandbruntime‚608  _runtime‚608hšÕ·ú .loss/FBt2/detection_loss‚7.509518682359605 _step‚111 %loss/total_loss‚88.99227920996778 /loss/FRt2/assignment_loss‚4.155434209309267 2loss/SRqqt1/assignment_loss‚2.3206451371792824 .loss/FRt2/detection_loss‚3.567067959000555 0loss/SRqqt1/detection_loss‚1.451324628251343 % _timestamp‚1.7858108633078349e+09 _runtime‚ 608.497638987 /loss/FRt1/detection_loss‚4.0684237043062845 epoch‚0 trainer/global_step‚1399 /loss/FBt1/detection_loss‚54.436337453623615 /loss/FRt1/assignment_loss‚5.351573638916015 1loss/SRqqt2/assignment_loss‚2.598752217988173 1loss/FBt1/assignment_loss‚0.38297061244763125 1loss/FBt2/assignment_loss‚0.03880885996234914 1loss/SRqqt2/detection_loss‚3.11142210662364964Pþ¸þ %loss/total_loss‚88.99227920996778 /loss/FRt2/assignment_loss‚4.155434209309267 2loss/SRqqt1/assignment_loss‚2.3206451371792824 .loss/FRt2/detection_loss‚3.567067959000555 0loss/SRqqt1/detection_loss‚1.451324628251343 % _timestamp‚1.7858108633078349e+09 _runtime‚ 608.497638987 /loss/FRt1/detection_loss‚4.0684237043062845 epoch‚0 trainer/global_step‚1399 /loss/FBt1/detection_loss‚54.436337453623615 /loss/FRt1/assignment_loss‚5.351573638916015 1loss/SRqqt2/assignment_loss‚2.598752217988173 1loss/FBt1/assignment_loss‚0.38297061244763125 1loss/FBt2/assignment_loss‚0.03880885996234914 1loss/SRqqt2/detection_loss‚3.1114221066236496 .loss/FBt2/detection_loss‚7.509518682359605 _step‚111oêÞ`¹‚(:’ µŸÅÓ¨¯ú‚ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚126.544' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚42.181333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0™»¦ÚTº‚(:É  µŸÅÓÅοƒ disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0# disk.nvme0n1.out‚535.7060546875 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0# proc.memory.rssMB‚ 11253.9453125 disk.drbd1356.in‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0$ memory_percent‚41.22183918952942* proc.memory.percent‚17.172157764434814 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.rbd1.out‚ 2.556640625 disk.sda.out‚ 18.64453125* proc.memory.availableMB‚38520.85546875 disk.dm-0.out‚535.7060546875 disk.drbd1552.out‚0 network.recv‚137492 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.drbd2111.out‚0 disk.drbd1673.in‚0 disk.drbd1931.out‚0 cpu‚14.153376295098159 disk.drbd2118.out‚0 disk.drbd1356.out‚0 disk.nvme0n1.in‚0 network.sent‚ 1.312478e+06* disk./.usagePercent‚32.858234939645534 disk.drbd1289.out‚0 disk.sda1.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚0.40625 disk.sda2.out‚ 18.64453125 disk.drbd1938.in‚0 disk.drbd1931.in‚0 disk.sda.in‚0.40625 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1584.in‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0$ disk./.usageGB‚587.5386123657227 disk.drbd2118.in‚0 disk.drbd2041.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚0 proc.cpu.threads‚29 disk.drbd1510.in‚0 disk.drbd1584.out‚0 disk.drbd1285.out‚0{û„0»+ _wandbruntime‚629  _runtime‚629¤sȼ ' lr-AdamW/pg1‚3.7343898217208536e-05 ' lr-AdamW/pg2‚3.7343898217208536e-05 trainer/global_step‚1449 % _timestamp‚1.7858108843905544e+09 _runtime‚ 629.580502814 _step‚112Aî¦Ì½Æ _runtime‚ 629.580502814 _step‚112 ' lr-AdamW/pg1‚3.7343898217208536e-05 ' lr-AdamW/pg2‚3.7343898217208536e-05 trainer/global_step‚1449 % _timestamp‚1.7858108843905544e+09pËÛh0¾+ _wandbruntime‚629  _runtime‚629÷‚7Mý¿¬O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5711045 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚25550259702108 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 $ _timestamp‚1.785810884391498e+09 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9636929 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15308707 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚26266453268992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24956143694848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15318363 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9870194 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 trainer/global_step‚1449 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚24896349041664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1275571708928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15308707 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚26171816281600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 O@DeviceâZêµ StatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1272448121276 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚24945634 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1275484279296 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚ 629.584633772 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1310284670464 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1275571708928 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4159147 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _step‚113 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚24896332002304 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5711047 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚26171920750592 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚24896332002304 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚25550156859624 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15318363 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24955292 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚24956168598528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚24277794541472Kœ‡¶'À°O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15318363 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9870194 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 trainer/global_step‚1449 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚24896349041664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1275571708928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15308707 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚26171816281600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1272448121276 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚24945634 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1275484279296 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚ 629.584633772 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1310284670464 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1275571708928 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4159147 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _step‚113 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚24896332002304 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5711047 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚26171920750592 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚24896332002304 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚25550156859624 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15318363 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24955292 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚24956168598528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚24277794541472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5711045 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚25550259702108 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 $ _timestamp‚1.785810884391498e+09 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9636929 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15308707 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚26266453268992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24956143694848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658qpå¤Æ0Á+ _wandbruntime‚629  _runtime‚629‰õ«½&·M JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 trainer/global_step‚1449 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚24913515804672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚24913498765312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1273239791374 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚26189967140352 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24962800 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9643572 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15319230 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1276451335680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24962800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚24973350811648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 629.932728659 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1276363898880 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1276451335680 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9643572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15319230 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚24972472 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚233 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15328900 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1311055905280 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚25567774505166 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1276363898880 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚24294534713792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24972472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9876664 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚24913515804672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚235 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24973375715328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E!nç…%ûM OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚26189862664192 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9643570 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚24913498765312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 629.932728659 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1276363898880 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1276451335680 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9643572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15319230 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚24972472 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚233 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15328900 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1311055905280 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚25567774505166 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1276363898880 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚24294534713792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24972472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9876664 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚24913515804672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚235 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24973375715328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24962800 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9643572 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15319230 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1276451335680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24962800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_ŽŒI<bytes.large_pool.freed‚24973350811648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1316336584784 trainer/global_step‚1499 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1319480414720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4301353 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚26431053939132 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15846365 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚25114820196832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1316250781660 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1319567844352 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5907897 % _timestamp‚1.7858109052503598e+09 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚27074150567424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚25816524438528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1319480414720 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚25754670152704 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚26431156781616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15836709 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9969079 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚27172070955008 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚25754670152704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680ôÛû¹'гO TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚27074150567424 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1355546516480 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚25805786 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1355424230912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9969079 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚25754687192064 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚25815444 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚25816499534848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10209502 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚ 650.443627473 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9969077 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15846365 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚27171923765760 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5907895 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4301605 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚25754687192064 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚27074150567424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚25816524438528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1319480414720 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚25754670152704 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚26431156781616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15836709 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9969079 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚27172070955008 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚25754670152704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1316336584784 trainer/global_step‚1499 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1319480414720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4301353 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚26431053939132 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15846365 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚25114820196832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1316250781660 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1319567844352 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5907897 % _timestamp‚1.7858109052503598e+09 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚25833731555328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚27092301426176 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚26448671584674 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25832624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15856902 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10216030 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15847232 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚27092196950016 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25822952 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5911834 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚27190190606848 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1320360034304 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10216297 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1317214058118 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9975722 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚27190026647552 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9975720 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800§ÛÅÂ&Ó¼M BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25832624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15856902 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10216030 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15847232 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚27092196950016 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25822952 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5911834 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚27190190606848 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1320360034304 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10216297 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1317214058118 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9975722 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚27190026647552 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9975720 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 _step‚118 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚25771853955072 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚25771853955072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4304463 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚25833731555328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚27092301426176 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚26448671584674 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640vøy÷ù0Ô+ _wandbruntime‚650  _runtime‚650küeÐÿÕù 1loss/FBt2/assignment_loss‚0.18917982347271833 /loss/FRt1/detection_loss‚3.8138969565741716 0loss/SRqqt2/detection_loss‚4.333123734167644 epoch‚0 trainer/global_step‚1499 /loss/FRt1/assignment_loss‚5.198020851612091 0loss/FBt1/assignment_loss‚0.3222404125789624 %loss/total_loss‚45.52067463975044 % _timestamp‚1.7858109056058455e+09 /loss/FBt1/detection_loss‚14.337838845679297 .loss/FBt2/detection_loss‚3.428321581179076 _runtime‚ 650.795634878 _step‚119 /loss/FRt2/assignment_loss‚4.203923016479335 1loss/SRqqt1/assignment_loss‚2.241360847766583 2loss/SRqqt2/assignment_loss‚2.6110628076962064 .loss/FRt2/detection_loss‚3.264357294003988 1loss/SRqqt1/detection_loss‚1.5773484685403627T€ïÖý /loss/FRt2/assignment_loss‚4.203923016479335 1loss/SRqqt1/assignment_loss‚2.241360847766583 2loss/SRqqt2/assignment_loss‚2.6110628076962064 .loss/FRt2/detection_loss‚3.264357294003988 1loss/SRqqt1/detection_loss‚1.5773484685403627 1loss/FBt2/assignment_loss‚0.18917982347271833 /loss/FRt1/detection_loss‚3.8138969565741716 0loss/SRqqt2/detection_loss‚4.333123734167644 epoch‚0 trainer/global_step‚1499 /loss/FRt1/assignment_loss‚5.198020851612091 0loss/FBt1/assignment_loss‚0.3222404125789624 %loss/total_loss‚45.52067463975044 % _timestamp‚1.7858109056058455e+09 /loss/FBt1/detection_loss‚14.337838845679297 .loss/FBt2/detection_loss‚3.428321581179076 _runtime‚ 650.795634878 _step‚119wÏÆ-L‘ׂ(:† âŸÅÓ²‘«ƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚116.403' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.801 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0+YÌESØ‚(:È  âŸÅÓ„ûìƒ disk.drbd2041.out‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.sda.in‚0.40625 disk.sda.out‚ 20.66015625 disk.drbd1226.in‚0 network.recv‚143908 disk.drbd2041.in‚0 disk.drbd1817.out‚0 disk.drbd1931.in‚0 disk.drbd1965.in‚0 disk.dm-0.out‚539.2021484375 disk.drbd1510.in‚0 cpu‚14.122898435868505 proc.cpu.threads‚29* disk./.usagePercent‚32.858220646010196$ memory_percent‚41.22198224067688 disk.drbd1226.out‚0 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0# disk.nvme0n1.out‚539.2021484375 disk.sda1.out‚0 disk.sda1.in‚0 network.sent‚ 1.40561e+06 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 20.66015625 disk.rbd1.in‚0 disk.drbd1584.out‚0 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0# proc.memory.rssMB‚ 11253.9453125$ disk./.usageGB‚587.5383567810059 disk.drbd1673.out‚0 disk.rbd1.out‚ 2.798828125 disk.drbd1931.out‚0 disk.drbd2078.out‚0* proc.memory.availableMB‚38520.76171875 disk.sda2.in‚0.40625 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.drbd1289.in‚0* proc.memory.percent‚17.172157764434814 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.dm-0.in‚0 disk.drbd2111.in‚0 disk.nvme0n1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd1584.in‚0"ª,K0Ù+ _wandbruntime‚671  _runtime‚671üÉ–ÈÚ _runtime‚ 671.809188718 _step‚120 ' lr-AdamW/pg1‚3.9921116865739145e-05 ' lr-AdamW/pg2‚3.9921116865739145e-05 trainer/global_step‚1549 % _timestamp‚1.7858109266192453e+09þ¤‹OÌÛÆ _runtime‚ 671.809188718 _step‚120 ' lr-AdamW/pg1‚3.9921116865739145e-05 ' lr-AdamW/pg2‚3.9921116865739145e-05 trainer/global_step‚1549 % _timestamp‚1.7858109266192453e+09x’ŠÀ\0Ü+ _wandbruntime‚671  _runtime‚671 ê¼O¹'ݳO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚26676880278528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚26676855374848 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚27312053861124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _step‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10301229 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10549758 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26665938 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10301227 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1363563979776 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚27976484853248 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26675596 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚28077438060544 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16364711 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1360225048292 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16374367 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚27311951018640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚27976484853248 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1363476550144 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚26613008303104 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 _runtime‚ 671.813646041 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4444759 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚25951828812832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚26613025342464 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚26613008303104 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10301229 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1400582685696 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 trainer/global_step‚1549 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1363476550144 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚27976589322240 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16364711 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10301227 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚27976589322240 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚25951811773472 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16374367 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10549504 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6104747 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10549758 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26665938 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10301227 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1363563979776 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚27976484853248 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26675596 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚28077438060544 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16364711 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1360225048292 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16374367 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚27311951018640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚27976484853248 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1363476550144 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚26613008303104 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 _runtime‚ 671.813646041 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4444759 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640y.@Ý0ß+ _wandbruntime‚672  _runtime‚672ZL-yÂ&à¼M TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚25968551945792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26692776 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚26630175066112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10307872 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26683104 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16384904 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10556207 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1364356169728 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16375234 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚28095546308096 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚26694087395328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚1549 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1364443606528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4447793 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚27329671506778 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26692776 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6108682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚26630175066112 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16384904 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚270 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚26694062491648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1401618677760 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚27329568664182 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4447525 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1364443606528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚268 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10307870 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚27994635712000 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚26630192105472dz”•Æ&áÀM R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚27994531235840 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1364356169728 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10307870 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 _step‚122 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1361016718390 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚26630192105472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1361102521626 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚1549 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1364443606528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4447793 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚27329671506778 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26692776 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6108682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚26630175066112 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16384904 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚270 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚26694062491648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1401618677760 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚27329568664182 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4447525 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1364443606528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚268 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10307870 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚27994635712000 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚26630192105472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚25968551945792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26692776 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚26630175066112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10307872 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26683104 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16384904 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10556207 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1364356169728 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16375234 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚28095546308096 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚26694087395328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚27471363492864 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚27471363492864 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4588649 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚28878923608064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10633379 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚28878923608064 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚28878819139072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚27471346453504 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27526090 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚28982985623040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16892713 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16892713 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1407472685568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1445774408192 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚28192950940632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚27471346453504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16902369 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10890002 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚28983143298048 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16902369 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1445907179520 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚26788837428832 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚28878819139072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10633377 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1407472685568 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚125 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10633379 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6301597 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7858109479717977e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1407560115200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 trainer/global_step‚1599 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1407560115200 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10633377 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚27537211214848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0©ãuñ½'î·O ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10633379 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚28878923608064 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚28878819139072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚27471346453504 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27526090 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚28982985623040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16892713 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16892713 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1407472685568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1445774408192 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚28192950940632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚27471346453504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16902369 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10890002 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚28983143298048 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16902369 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1445907179520 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚26788837428832 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚28878819139072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10633377 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1407472685568 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚125 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10633379 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6301597 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7858109479717977e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1407560115200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 trainer/global_step‚1599 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1407560115200 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10633377 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚27537211214848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚244 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚242 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚27537236118528 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚27471363492864 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚27471363492864 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4588649 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚28878923608064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18}6mä™0ï+ _wandbruntime‚693  _runtime‚693ÐòëÑÂ&ð¼M L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16903236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚27488530255872 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27552928 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1446679241728 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1408439741952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4591504 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1446809908736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 _step‚126 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27552928 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4591260 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27543256 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1408352305152 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚28210465743690 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10640022 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10896792 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚29001097573376 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚27554418331648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16903236 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 _runtime‚ 693.519169175 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6305534 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚27554443235328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1404905181898 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16912906 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚28210568586286 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10640020 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1408352305152 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚28896865521664 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚29001253144064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121}¿ ƒË ñÀM FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1408439741952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4591504 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1446809908736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 _step‚126 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27552928 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4591260 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27543256 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1408352305152 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚28210465743690 JBDeviceStatsMonitor.on_train¹ç‘ºû_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10640022 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10896792 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚29001097573376 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚27554418331648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16903236 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 _runtime‚ 693.519169175 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6305534 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚27554443235328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1404905181898 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16912906 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚28210568586286 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10640020 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1408352305152 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚28896865521664 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚29001253144064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16903236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚27488530255872 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27552928 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1446679241728 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚29781257893888 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10965527 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚29888492700672 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10965527 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚28329684603904 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚29781153424896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1451556250624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1451468820992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1451556250624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17420715 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28395900 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚27625846044832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4731804 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚28329701643264 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚28397591958528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1490925645824 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1448001975308 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28386242 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚27625829005472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1447916172184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚1649Ê$  ¾'ü¸O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚29781153424896 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚28329701643264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚29781257893888 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10965527 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚29888492700672 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10965527 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚28329684603904 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚29781153424896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1451556250624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1451468820992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1451556250624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17420715 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28395900 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚27625846044832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4731804 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚28329701643264 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚28397591958528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1490925645824 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1448001975308 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28386242 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚27625829005472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1447916172184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚1649 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _runtime‚ 714.409640112 _step‚129 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚28413080 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1452348440576 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28413080 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚29091465665794 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4734567 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚28414774171648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1448793645406 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚28346851366912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28403408 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6502384 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚28346868406272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1452435877376 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10972172 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1452348440576 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚237 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚235 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚29906761065472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6502382 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚27642569177792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚28346868406272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11236714 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1491961990144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4734332 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17431238 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17431238 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0ãF˜È&ÿÂM YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚27642569177792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚28346868406272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11236714 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1491961990144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4734332 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17431238 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17431238 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚27642586217152 % _timestamp‚1.7858109695690463e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10972170 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚29799199807488 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10972170 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚28414799075328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1452435877376 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚28413080 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1452348440576 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28413080 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚29091465665794 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4734567 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚28414774171648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1448793645406 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚28346851366912 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28403408 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6502384 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚28346868406272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1452435877376 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10972172 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1452348440576 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚237 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚235 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚29906761065472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6502382 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0‚Öýø0€+ _wandbruntime‚714  _runtime‚714KPôú epoch‚0 0loss/FRt2/assignment_loss‚3.9697951820161608 1loss/SRqqt1/detection_loss‚1.3069822780639015 0loss/SRqqt2/detection_loss‚3.263149640031075 .loss/FBt2/detection_loss‚5.676192658191958 %loss/total_loss‚44.52633624929028 % _timestamp‚1.7858109695702822e+09 _step‚131 2loss/SRqqt1/assignment_loss‚1.9056710704513218 .loss/FRt1/detection_loss‚4.360856822133064 0loss/FBt1/assignment_loss‚0.2952050113348047 .loss/FRt2/detection_loss‚3.469991324676408 /loss/FBt1/detection_loss‚13.178786589912312 0loss/FRt1/assignment_loss‚4.7603660549436295 1loss/SRqqt2/assignment_loss‚2.217956122387661 1loss/FBt2/assignment_loss‚0.12138349514798788 trainer/global_step‚1649 _runtime‚ 714.760070727…J”¼‚ÿ % _timestamp‚1.7858109695702822e+09 _step‚131 2loss/SRqqt1/assignment_loss‚1.9056710704513218 .loss/FRt1/detection_loss‚4.360856822133064 0loss/FBt1/assignment_loss‚0.2952050113348047 .loss/FRt2/detection_loss‚3.469991324676408 /loss/FBt1/detection_loss‚13.178786589912312 0loss/FRt1/assignment_loss‚4.7603660549436295 1loss/SRqqt2/assignment_loss‚2.217956122387661 1loss/FBt2/assignment_loss‚0.12138349514798788 trainer/global_step‚1649 _runtime‚ 714.760070727 epoch‚0 0loss/FRt2/assignment_loss‚3.9697951820161608 1loss/SRqqt1/detection_loss‚1.3069822780639015 0loss/SRqqt2/detection_loss‚3.263149640031075 .loss/FBt2/detection_loss‚5.676192658191958 %loss/total_loss‚44.52633624929028ƒdÖ«Vƒ‚(:Ë  ž Åӌ̃ disk.drbd1938.out‚0 disk.drbd1931.in‚0 disk.drbd1552.out‚0 proc.cpu.threads‚29 network.sent‚ 1.543692e+06 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.drbd2078.out‚0 disk.drbd1673.out‚0% memory_percent‚41.221851110458374 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.drbd1285.in‚0 disk.sda2.out‚ 22.47265625* proc.memory.percent‚17.172157764434814 disk.dm-0.in‚0 disk.drbd1817.out‚0 disk.drbd1584.in‚0 disk.drbd1550.out‚0 disk.drbd1289.in‚0 disk.drbd1828.out‚0 disk.rbd0.out‚0 disk.sda1.in‚0$ disk.nvme0n1.out‚540.16748046875 disk.drbd1965.in‚0! disk.dm-0.out‚540.16748046875 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚0.40625 disk.rbd1.in‚0 disk.rbd0.in‚0 disk.drbd1965.out‚0# proc.memory.rssMB‚ 11253.9453125 cpu‚14.086035110200509 disk.drbd2118.out‚0* proc.memory.availableMB‚38520.84765625 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0$ disk./.usageGB‚587.5388603210449 disk.drbd1673.in‚0 disk.rbd1.out‚ 3.017578125 disk.nvme0n1.in‚0 disk.drbd1584.out‚0 disk.drbd1289.out‚0 network.recv‚152410) disk./.usagePercent‚32.85824880660518 disk.sda.in‚0.40625 disk.sda.out‚ 22.47265625Zzò<‘„‚(:† ž ÅÓºªÕƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.089' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚42.363 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0GÈg$œ…‚(:‘ ­ ÅÓö¤¥ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚128.486' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.82866666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0U–èU†‚(:Ê  ­ ÅÓ£©ïƒ disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd2078.in‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.drbd1552.out‚0 network.sent‚ 1.5886e+06 network.recv‚154860 disk.nvme0n1.in‚0$ disk.nvme0n1.out‚540.36669921875 disk.sda.out‚ 22.82421875 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.sda.in‚0.40625 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.drbd1226.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.sda2.in‚0.40625 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd1289.out‚0 disk.drbd2118.out‚0 disk.drbd1584.out‚0 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd1828.in‚0* proc.memory.percent‚17.172163724899292 disk.dm-0.in‚0$ proc.memory.rssMB‚11253.94921875$ disk./.usageGB‚587.5408248901367! disk.dm-0.out‚540.36669921875 proc.cpu.threads‚29 disk.sda2.out‚ 22.82421875 disk.drbd1931.out‚0 disk.rbd0.in‚0* proc.memory.availableMB‚38520.54296875 cpu‚14.078455922610974 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd2041.out‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0 disk.drbd1938.out‚0) disk./.usagePercent‚32.85835867559322 disk.rbd1.in‚0 disk.drbd1356.in‚0 disk.drbd1584.in‚0 disk.drbd1828.out‚0$ memory_percent‚41.22231602668762 disk.rbd1.out‚ 3.0498046875 disk.drbd1938.in‚0loe0‡+ _wandbruntime‚735  _runtime‚735ãÄJ>ň¿ & lr-AdamW/pg1‚4.378694483853506e-05 & lr-AdamW/pg2‚4.378694483853506e-05 trainer/global_step‚1699 % _timestamp‚1.7858109906383662e+09 _runtime‚ 735.82831138 _step‚132üÿDÊ‰Ä _step‚132 & lr-AdamW/pg1‚4.378694483853506e-05 & lr-AdamW/pg2‚4.378694483853506e-05 trainer/global_step‚1699 % _timestamp‚1.7858109906383662e+09 _runtime‚ 735.82831138„vÒ0Š+ _wandbruntime‚735  _runtime‚735›ß<µ ‹³O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚28462837621472 % _timestamp‚1.7858109906393204e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚29257947798528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17958373 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11297677 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4873849 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29246394 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚29954642257164 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚ 735.832641261 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1495552386048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚236 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚30794037547520 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1536245326848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_ûCÐS³pool.allocated‚121 trainer/global_step‚1699 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚30683487710720 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17948717 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17958373 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚29257922894848 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚28462854660832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1491890438816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚29188039793664 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚29954745099648 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17948717 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚30683592179712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1495464956416 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11297679ÖFM>¾'Œ¸O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17948717 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚30683592179712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1495464956416 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11297679 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚28462837621472 % _timestamp‚1.7858109906393204e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚29257947798528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17958373 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11297677 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4873849 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29246394 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚29954642257164 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚ 735.832641261 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1495552386048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚236 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚30794037547520 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1536245326848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 trainer/global_step‚1699 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚30683487710720 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17948717 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17958373 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚29257922894848 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚28462854660832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1491890438816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚29188039793664 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚29954745099648 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121…ù“p0+ _wandbruntime‚736  _runtime‚736³•ÄÂ&Ž¼M R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚30701534093312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1496344576000 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1496344576000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11304320 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 _runtime‚ 736.19265817 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1492767912150 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚29205189517312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11576185 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚30812158451712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚1699 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29273232 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6699234 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚29275154915328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1492682108914 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚30701638569472 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17959240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29273232 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17959240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚134 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚29205206556672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4876951 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11304322 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1537028440064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17968910 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚30701638569472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚29275130011648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6699232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚29205206556672 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11575939 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1496432012800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚29972259902706 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11304322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1537163301376 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858109910023148e+09 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936¶»Ç&ÁM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1496432012800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚29972362745302 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11304320 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚29263560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚28479577793792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17968910 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4876707 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚30701534093312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1496344576000 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1496344576000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11304320 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 _runtime‚ 736.19265817 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1492767912150 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚29205189517312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11576185 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚30812158451712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚1699 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29273232 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6699234 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚29275154915328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1492682108914 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚30701638569472 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17959240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29273232 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17959240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚134 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚29205206556672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4876951 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11304322 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1537028440064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17968910 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚30701638569472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚29275130011648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6699232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚29205206556672 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11575939 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1496432012800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚29972259902706 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11304322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1537163301376 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858109910023148e+09 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚247 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚30835642179156 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚29299863276832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1539461091840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚1749 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚30046377944064 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚30118303638528 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11909257 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1581457017856 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚30118278734848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚31585926465536 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1539548521472 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6892147 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11629827 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚30046360904704 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18486375 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11629829 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚31699760656384 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _step‚137 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚31585926465536 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18476719 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚31585821996544 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1581317955072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1539461091840 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚31699596689920 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚31585821996544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1539548521472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11909008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1535693099200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚249 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚31585926465536 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1539548521472 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6892147 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11629827 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚30046360904704 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18486375 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11629829 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚31699760656384 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _step‚137 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚31585926465536 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18476719 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚31585821996544 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1581317955072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1539461091840 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚31699596689920 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚31585821996544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1539548521472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11909008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1535693099200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚249 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚247 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚30835642179156 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚29299863276832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1539461091840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚1749 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚30046377944064 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚30118303638528 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11909257 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548‰ˆçM0›+ _wandbruntime‚757  _runtime‚7570J ÿœ"œ½M REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18496912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1582220363776 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1540340711424 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11915733 trainer/global_step‚1749 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚31603972855296 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚30063544707072 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1540340711424 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚31603868379136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11636470 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11915973 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11636470 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚30063544707072 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1536570572422 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5019889 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30135485851648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚30853156982214 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚29316603449152 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚31717706215424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1582361516544 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11636472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30133384 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18487242 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6896084 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5019651 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚238 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _step‚138 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚31603972855296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 141152768 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18487242 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 % _timestamp‚1.7858110121797323e+09 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_en'd/allocation.all.allocated‚30133384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1540428148224 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11636472 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30133384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1540428148224 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11636472 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11915733 trainer/global_step‚1749 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚31603972855296 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚30063544707072 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1540340711424 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚31603868379136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11636470 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11915973 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11636470 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚30063544707072 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1536570572422 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5019889 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30135485851648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚30853156982214 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚29316603449152 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚31717706215424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1582361516544 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11636472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30133384 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18487242 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6896084 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5019651 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚238 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _step‚138 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚31603972855296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 141152768 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18487242 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 % _timestamp‚1.7858110121797323e+09Š8Ç¡˜0ž+ _wandbruntime‚757  _runtime‚757¡g>?üŸö /loss/FRt1/assignment_loss‚4.702507661867745 2loss/SRqqt1/assignment_loss‚1.7643810635007273 1loss/SRqqt2/assignment_loss‚2.014374399688882 /loss/FRt2/detection_loss‚3.7283368539519426 0loss/SRqqt2/detection_loss‚4.058612398278545 %loss/total_loss‚53.05581921293779 trainer/global_step‚1749 epoch‚0 _step‚139 .loss/FRt2/assignment_loss‚4.06182564758673 0loss/FBt1/assignment_loss‚0.3190843810235882 /loss/FBt2/assignment_loss‚0.142633374062522 .loss/FRt1/detection_loss‚3.702223131362396 1loss/SRqqt1/detection_loss‚1.4666392806754969 /loss/FBt1/detection_loss‚19.204148848141944 .loss/FBt2/detection_loss‚7.891052172797269 % _timestamp‚1.7858110121809556e+09 _runtime‚ 757.370744981ŠïÉà û 0loss/SRqqt2/detection_loss‚4.058612398278545 %loss/total_loss‚53.05581921293779 trainer/global_step‚1749 epoch‚0 _step‚139 .loss/FRt2/assignment_loss‚4.06182564758673 0loss/FBt1/assignment_loss‚0.3190843810235882 /loss/FBt2/assignment_loss‚0.142633374062522 .loss/FRt1/detection_loss‚3.702223131362396 1loss/SRqqt1/detection_loss‚1.4666392806754969 /loss/FBt1/detection_loss‚19.204148848141944 .loss/FBt2/detection_loss‚7.891052172797269 % _timestamp‚1.7858110121809556e+09 _runtime‚ 757.370744981 /loss/FRt1/assignment_loss‚4.702507661867745 2loss/SRqqt1/assignment_loss‚1.7643810635007273 1loss/SRqqt2/assignment_loss‚2.014374399688882 /loss/FRt2/detection_loss‚3.7283368539519426‹´=ÊW¡‚(:Ì  Ë ÅÓá¿Ïƒ disk.dm-0.out‚ 541.302734375 disk.drbd1510.out‚0 disk.drbd1356.out‚0 disk.drbd1931.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 3.2060546875* proc.memory.percent‚17.172211408615112 network.sent‚ 1.63656e+06 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd1938.out‚0 disk.drbd2041.out‚0 disk.drbd2111.in‚0 disk.drbd1673.in‚0 proc.cpu.threads‚29 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.drbd1289.in‚0 disk.sda2.out‚ 24.19921875 disk.drbd1965.out‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.out‚0 network.recv‚158238( proc.memory.availableMB‚ 38520.640625 disk.dm-0.in‚0$ proc.memory.rssMB‚11253.98046875 disk.rbd0.in‚0 disk.drbd1510.in‚0 cpu‚14.062791241242138 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1289.out‚0 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0" disk.nvme0n1.out‚ 541.302734375 disk.drbd1550.in‚0 disk.drbd1817.in‚0 disk.drbd1673.out‚0 disk.drbd2041.in‚0 disk.sda.out‚ 24.19921875 disk.drbd1965.in‚0 disk.drbd2078.in‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1828.in‚0 disk.drbd1356.in‚0$ disk./.usageGB‚586.6030616760254 disk.drbd1550.out‚0 disk.drbd1817.out‚0) disk./.usagePercent‚32.80591404751515 disk.drbd1226.in‚0 disk.drbd1285.out‚0 disk.rbd1.in‚0 disk.drbd1584.in‚0% memory_percent‚41.222167015075684‡þ¸„‘¢‚(:† Ë ÅÓ€ÌɃ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚126.057' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚42.019 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0LÔÑ:0£+ _wandbruntime‚778  _runtime‚778tž0ÔƤÀ & lr-AdamW/pg2‚4.636416348706567e-05 trainer/global_step‚1799 % _timestamp‚1.7858110330045543e+09 _runtime‚ 778.194563219 _step‚140 & lr-AdamW/pg1‚4.636416348706567e-05“L¢Ë¥Å & lr-AdamW/pg1‚4.636416348706567e-05 & lr-AdamW/pg2‚4.636416348706567e-05 trainer/global_step‚1799 % _timestamp‚1.7858110330045543e+09 _runtime‚ 778.194563219 _step‚140Œüô=-0¦+ _wandbruntime‚778  _runtime‚778OûsǸ'§²O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚32488156282368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12248422 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚30136854853472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚30904699055104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1583544656896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11961979 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚32488260751360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12248172 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7088995 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚32605149278720 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚30904716094464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7088997 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5159177 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚30904716094464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19004721 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚30978659478528 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚30904699055104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11961979 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11961977 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚32488260751360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19014377 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1579581562708 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1626641183744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11961977 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1583544656896 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚30978634574848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚32605300662272 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚32488156282368 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5159425 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1626514703872 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚30976356 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 trainer/global_step‚1799«™Äø¨·O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚32488156282368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12248422 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚30136854853472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚30904699055104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1583544656896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11961979 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚32488260751360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12248172 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7088995 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚32605149278720 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚30904716094464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7088997 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5159177 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚30904716094464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19004721 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚30978659478528 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚30904699055104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11961979 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11961977 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚32488260751360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19014377 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1579581562708 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1626641183744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11961977 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1583544656896 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚30978634574848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚32605300662272 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚32488156282368 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5159425 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1626514703872 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚30976356 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 trainer/global_step‚1799,• 0©+ _wandbruntime‚778  _runtime‚778UåqÞÂ&ª¼M REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1580459035930 trainer/global_step‚1799 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚32506307141120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19024914 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7092934 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11968620 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1584424283648 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚30921865818112 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5162045 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _step‚142 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30993536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30993536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚30921865818112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 $ _timestamp‚1.785811033357853e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11968622 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚30983864 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚32506202664960 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1584424283648 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚31734054061722 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12254977 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19015244 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚32623412072960 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1580544839166 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12255232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1584336846848 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11968622 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19015244 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚30153612065152 _runtime‚ 778.548148041 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1584336846848 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚31734156904318 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚30921882857472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30995841691648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚32623256502272 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1627545477632ÖýfÕÇ&«ÁM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1584424283648 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚30921865818112 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5162045 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _step‚142 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30993536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30993536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚30921865818112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 $ _timestamp‚1.785811033357853e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11968622 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚30983864 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚32506202664960 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1584424283648 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚31734054061722 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12254977 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19015244 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚32623412072960 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1580544839166 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12255232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1584336846848 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11968622 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19015244 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚30153612065152 _runtime‚ 778.548148041 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1584336846848 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚31734156904318 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚30921882857472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30995841691648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚32623256502272 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1627545477632 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1580459035930 trainer/global_step‚1799 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚32506307141120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19024914 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7092934 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11968620 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _step‚145 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚33390490568192 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19542379 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12294127 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1627453362688 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚33510690116096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12294129 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1623555829340 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚31763037205504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚31763037205504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1627453362688 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1627540792320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚31838990414848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚33390490568192 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5302739 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚32597333495688 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚263 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7285845 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚33390595037184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚31826850 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12294129 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12588323 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1671699701248 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1671836666880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19532723 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚33390595037184 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚31839015318528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚31763054244864 ãܾ½'¸·O 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1627453362688 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚33510690116096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12294129 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1623555829340 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚31763037205504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚31763037205504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1627453362688 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1627540792320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚31838990414848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚33390490568192 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5302739 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚32597333495688 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚263 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7285845 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚33390595037184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚31826850 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12294129 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12588323 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1671699701248 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1671836666880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19532723 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚33390595037184 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚31839015318528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚31763054244864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _step‚145 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚33390490568192 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19542379 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12294127 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200‘ê²0¹+ _wandbruntime‚799  _runtime‚799øf-­Â&º¼M JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚33408641426944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚31853688 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚33528804474880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚31780221007872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19543246 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1672735513088 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚31844016 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1624347499438 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 799.681941318 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚33528957948416 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31853688 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19552916 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12595156 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚1849 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1628332982272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5305629 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19543246 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12300772 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚31780221007872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5305374 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚32615053983826 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12300770 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚33408641426944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1628420419072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚30990620681152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19552916 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12595413 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7289784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12300772 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12300770 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚31844016 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚31856222435328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1672606943232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚31856197531648 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚32614951141230 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1624433302674 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚33408536950784 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 $ _timestamp‚1.785811054491662e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236Õ¥‘÷€»ÁM ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 $ _timestamp‚1.785811054491662e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚33408641426944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚31853688 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚33528804474880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚31780221007872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19543246 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1672735513088 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚31844016 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1624347499438 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 799.681941318 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚33528957948416 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31853688 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19552916 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12595156 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚1849 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1628332982272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5305629 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19543246 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12300772 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚31780221007872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5305374 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚32615053983826 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12300770 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚33408641426944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1628420419072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚30990620681152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19552916 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12595413 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7289784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12300772 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12300770 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚31844016 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚31856222435328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1672606943232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚31856197531648 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚32614951141230 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1624433302674 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚33408536950784 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024’®Ê^0¼+ _wandbruntime‚799  _runtime‚799½?mðý½÷ 1loss/SRqqt2/detection_loss‚3.3675911821598232 epoch‚0 % _timestamp‚1.7858110544928553e+09 /loss/FRt2/detection_loss‚3.1394834545802097 -loss/FBt2/detection_loss‚8.73133788284913 $loss/total_loss‚50.1017231894139 trainer/global_step‚1849 _runtime‚ 799.682649 2loss/SRqqt1/assignment_loss‚1.6672370278202326 .loss/FRt1/detection_loss‚4.240308766636779 _step‚147 0loss/FRt2/assignment_loss‚4.0739895081033515 0loss/FBt1/assignment_loss‚0.5433651775529142 1loss/FBt2/assignment_loss‚0.27060789381413575 /loss/FBt1/detection_loss‚15.614208722925362 /loss/FRt1/assignment_loss‚4.811157310710234 2loss/SRqqt2/assignment_loss‚2.0626794255750123 1loss/SRqqt1/detection_loss‚1.5797568366867223ÏáÙÔ¾ü /loss/FRt2/detection_loss‚3.1394834545802097 -loss/FBt2/detection_loss‚8.73133788284913 $loss/total_loss‚50.1017231894139 trainer/global_step‚1849 _runtime‚ 799.682649 2loss/SRqqt1/assignment_loss‚1.6672370278202326 .loss/FRt1/detection_loss‚4.240308766636779 _step‚147 0loss/FRt2/assignment_loss‚4.0739895081033515 0loss/FBt1/assignment_loss‚0.5433651775529142 1loss/FBt2/assignment_loss‚0.27060789381413575 /loss/FBt1/detection_loss‚15.614208722925362 /loss/FRt1/assignment_loss‚4.811157310710234 2loss/SRqqt2/assignment_loss‚2.0626794255750123 1loss/SRqqt1/detection_loss‚1.5797568366867223 1loss/SRqqt2/detection_loss‚3.3675911821598232 epoch‚0 % _timestamp‚1.7858110544928553e+09“½H#›¿‚(: ø ÅÓļʃ gpu.0.gpu‚37.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚95.183' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚31.72766666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ó²ƒ•HÀ‚(:½  ø ÅÓΩۃ disk.sda.in‚ 0.41015625 disk.drbd2078.in‚0 disk.drbd1938.out‚0 cpu‚14.041511751737065 network.recv‚164233 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.drbd1584.in‚0 disk.drbd1226.in‚0 disk.drbd2118.in‚0 disk.drbd1931.in‚0 disk.drbd1965.out‚0 disk.sda2.out‚25.25 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.sda1.in‚0 network.sent‚ 1.72943e+06$ disk./.usageGB‚586.6028060913086" disk.nvme0n1.out‚ 552.029296875 disk.drbd1965.in‚0& proc.memory.availableMB‚ 38520.4375 disk.drbd2041.out‚0 disk.drbd2111.in‚0 disk.drbd2118.out‚0 disk.sda2.in‚ 0.41015625 disk.drbd1226.out‚0 disk.drbd1289.in‚0 disk.drbd1673.in‚0 disk.rbd1.out‚ 3.23828125* proc.memory.percent‚17.172211408615112 disk.drbd1938.in‚0* disk./.usagePercent‚32.805899753879814 disk.drbd1356.out‚0 disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11253.98046875 disk.sda.out‚25.25 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.dm-0.in‚0 proc.cpu.threads‚29 disk.drbd1828.out‚0 disk.sda1.out‚0 disk.drbd1356.in‚0 disk.drbd1817.out‚0% memory_percent‚41.222476959228516 disk.drbd1285.in‚0 disk.drbd1510.out‚0 disk.rbd0.in‚0 disk.drbd1510.in‚0 disk.nvme0n1.in‚0 disk.dm-0.out‚ 552.029296875 disk.drbd1552.out‚0M080Á+ _wandbruntime‚820  _runtime‚820㽉øÆÂÀ $ _timestamp‚1.785811075611362e+09 _runtime‚ 820.80131681 _step‚148 ' lr-AdamW/pg1‚4.8941382135596276e-05 ' lr-AdamW/pg2‚4.8941382135596276e-05 trainer/global_step‚1899w_½œËÃÅ trainer/global_step‚1899 $ _timestamp‚1.785811075611362e+09 _runtime‚ 820.80131681 _step‚148 ' lr-AdamW/pg1‚4.8941382135596276e-05 ' lr-AdamW/pg2‚4.8941382135596276e-05”¶mÜ/0Ä+ _wandbruntime‚820  _runtime‚820Îi½¹'ųO VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12626277 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚32621392395264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1716864986112 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1671536927744 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20070381 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12928135 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚32699371158528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚1899 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1716999854592 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚31810889124832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12626279 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚32621392395264 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20060725 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚33478230575196 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1667444292848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚32699346254848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1671536927744 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚33478333417680 _runtime‚ 820.805656875 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1671449498112 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12626277 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12928370 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20060725 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚32621375355904 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _step‚149 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚34292824854016 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1671449498112 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32696660 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚34292824854016 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858110756123183e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32687002 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚34292929323008 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12626279 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5445673 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1667358489724 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20070381 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚34416211240960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚32621375355904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7482697 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚34292929323008 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7482695 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚233 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚235 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5445440 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚34416371013120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200Áÿ ¾'ƸO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7482695 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚233 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚235 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5445440 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚34416371013120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12626277 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚32621392395264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1716864986112 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1671536927744 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20070381 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12928135 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚32699371158528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚1899 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1716999854592 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚31810889124832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12626279 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚32621392395264 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20060725 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚33478230575196 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1667444292848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚32699346254848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1671536927744 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚33478333417680 _runtime‚ 820.805656875 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1671449498112 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12626277 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12928370 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20060725 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚32621375355904 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _step‚149 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚34292824854016 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1671449498112 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32696660 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚34292824854016 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858110756123183e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32687002 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚34292929323008 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12626279 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5445673 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1667358489724 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20070381 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚34416211240960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚32621375355904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7482697 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚34292929323008 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130•×À²40Ç+ _wandbruntime‚821  _runtime‚821 dz­È½M H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚32716578275328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚32638559158272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1672329117696 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚34310871236608 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚32638559158272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚150 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20080918 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7486632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚33495848220738 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12934870 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚33495951063334 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1668321766182 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32713840 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12632922 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5448238 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20071248 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5448488 trainer/global_step‚1899 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12632920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008ƉÈ&ÉÂM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7486632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚33495848220738 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12934870 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚33495951063334 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1668321766182 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32713840 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12632922 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5448238 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20071248 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5448488 trainer/global_step‚1899 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12632920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚32716578275328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚32638559158272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1672329117696 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚34310871236608 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚32638559158272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚150 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20080918 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ONe|0Ô+ _wandbruntime‚841  _runtime‚841N¤“Z¹'Õ³O WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7679547 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚153 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20598383 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _runtime‚ 841.475745715 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚32647880701472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20598383 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12958429 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12958427 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚33559726998528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚32647897740832 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7679545 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚34359127654704 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1762170735104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1715533063168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20588727 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚35321735864320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1711332756356 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12958427 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚33479713506304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚33479730545664 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1715533063168 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1762033769472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚35195263608832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13268469 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13268212 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33556812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚35195159139840 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1715445633536 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚35195263608832 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚35321897733632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20588727 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1715445633536 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5588667 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚33559702094848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚33479713506304 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12958429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858110962820961e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚33547154 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚35195159139840 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚33479730545664 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 trainer/global_step‚1949 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚34359127654704 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1762170735104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1715533063168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20588727 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚35321735864320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1711332756356 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12958427 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚33479713506304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚33479730545664 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1715533063168 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1762033769472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚35195263608832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13268469 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13268212 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33556812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚35195159139840 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1715445633536 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚35195263608832 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚35321897733632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20588727 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1715445633536 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5588667 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚33559702094848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚33479713506304 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12958429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858110962820961e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚33547154 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚35195159139840 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚33479730545664 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 trainer/global_step‚1949 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _runtime‚ 841.475745715 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚32647880701472™‘SîÇ0×+ _wandbruntime‚841  _runtime‚841‹IÛìÃ&ؽM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13275010 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1716412689920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1716412689920 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚35339844234752 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 trainer/global_step‚1949 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20599250 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20608920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12965072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7683482 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1762935023104 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚33496897308672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33573992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20599250 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33573992 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚33576934115328 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚35340006096896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚33496880269312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13275245 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚35213309998592 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚32664620873792 % _timestamp‚1.7858110966354246e+09 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚33576909211648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚33496897308672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33564320 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚33496880269312 _runtime‚ 841.825720645 _step‚154 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚233 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12965070 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚235 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5591761 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1763071981568 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚32664637913152¦&Ð/È&ÙÂM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚35213309998592 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1716325253120 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚34376848142842 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚235 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5591761 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1763071981568 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚32664637913152 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13275010 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1716412689920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1716412689920 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚35339844234752 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 trainer/global_step‚1949 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20599250 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20608920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12965072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7683482 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1762935023104 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚33496897308672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33573992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20599250 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33573992 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚33576934115328 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚35340006096896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚33496880269312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13275245 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚35213309998592 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚32664620873792 % _timestamp‚1.7858110966354246e+09 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚33576909211648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚33496897308672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33564320 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚33496880269312 _runtime‚ 841.825720645 _step‚154 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚233 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12965070 ODeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚36097493425664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13290577 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚34338051656704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21126385 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚34420082838528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34407306 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1759529198592 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1759441768960 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚35240127576696 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1755135416740 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚36097493425664 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5732342 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚34338051656704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13608996 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1807163984384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚33484889317472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚ 862.970629466 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _step‚157 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13290577 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7876397 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚36097597894656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13290579 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚36097597894656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13608737 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936¦àòi½'ä·O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13608737 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚1999 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚36097493425664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13290577 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚34338051656704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21126385 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚34420082838528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34407306 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1759529198592 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1759441768960 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚35240127576696 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1755135416740 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚36097493425664 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5732342 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚34338051656704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13608996 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1807163984384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚33484889317472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚ 862.970629466 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _step‚157 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13290577 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7876397 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚36097597894656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13290579 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚36097597894656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712·Í00å+ _wandbruntime‚863  _runtime‚863‚S,Ã&æ½M YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1760408825344 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1760408825344 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚34355218419712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1760321388544 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚ 863.319647793 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5735392 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚36245503612416 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚33501646529152 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21136922 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13297222 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚35257642379754 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13297220 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1760321388544 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7880334 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1808213657088 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚36115539808256 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚34355218419712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚33501629489792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1756012889962 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚35257745222350 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34434144 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13297220 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚1999 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13615481 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1808078795776 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚34437265051648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5735149 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚34355235459072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251‘ígçÂM REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13297222 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13615481 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1808078795776 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚34437265051648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5735149 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚34355235459072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1760408825344 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1760408825344 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚34355218419712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1760321388544 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 -Ì‚ÀIMGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚ 863.319647793 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5735392 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚36245503612416 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚33501646529152 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21136922 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13297222 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚35257642379754 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13297220 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1760321388544 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7880334 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1808213657088 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚36115539808256 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚34355218419712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚33501629489792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1756012889962 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚35257745222350 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34434144 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13297220 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚1999 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21654387 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚36999827711488 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚35196406846464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1852361030144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1799109683372 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚35196406846464 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚36999932180480 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚35196389807104 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1803437904384 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚35280438678528 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13622729 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚36999932180480 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 884.079082902 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13948570 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _step‚161 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13622729 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚34321914972832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1852497995776 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13622727 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚36999827711488 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚2049 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1803525334016 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200N-ð„¾'ò¸O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21644731 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚37132774804992 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚36120921813720 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚35267458 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13948310 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚34321897933472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚35196389807104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5875065 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚37132936674304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35277116 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5875323 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1803525334016 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚35280413774848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13622727 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1803437904384 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚35280438678528 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13622729 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚36999932180480 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 884.079082902 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13948570 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _step‚161 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13622729 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚34321914972832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1852497995776 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13622727 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚36999827711488 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚2049 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1803525334016 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21654387 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚36121024656204 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8073247 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1799023880248 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1803437904384 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21644731 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21654387 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚36999827711488 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚35196406846464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1852361030144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1799109683372 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚35196406846464 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚36999932180480 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚35196389807104 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1799987156706 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚35294296 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35294296 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1853397300736 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚35213573609472 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚36138539459262 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1799901353470 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21655254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚35213556570112 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13629370 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚35297620891648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37017978570240 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5877897 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚162 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21664924 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚34338655145152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13629372 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚37151043096064 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚36138642301858 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13629372 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚35213556570112 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13955079 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8077184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1853268730880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚34338638105792 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8077182 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚35213573609472 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785811139247208e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1804404960768 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1804317523968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37017978570240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1804317523968 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5878142 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚35284624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21664924 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13629370 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37017874094080 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1804404960768 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚35297645795328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121Ô/# Ç&õÁM H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35294296 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1853397300736 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚35213573609472 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚36138539459262 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1799901353470 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21655254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚35213556570112 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13629370 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚35297620891648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37017978570240 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5877897 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚162 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21664924 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚34338655145152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13629372 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚37151043096064 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚36138642301858 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13629372 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚35213556570112 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13955079 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8077184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1853268730880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚34338638105792 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8077182 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚35213573609472 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785811139247208e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1804404960768 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1804317523968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37017978570240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1804317523968 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5878142 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚35284624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21664924 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13629370 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37017874094080 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1804404960768 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚35297645795328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21655254 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚37150889622528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚2049 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35284624 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1799987156706 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚35294296 ODeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1842912343756 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚38038301885952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1842998146880 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚38038461658112 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚37902266466304 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚37902266466304 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚36140769614848 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚36140794518528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚35158906549472 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22182389 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1847521469440 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚37001921735712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22182389 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚236 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚36054744996864 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚36054727957504 trainer/global_step‚2099 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13954879 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚37902161997312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚37001818893228 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚36054727957504 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22172733 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8270097 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856•Yz§¾'‚¸O ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚35158923588832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚37902161997312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚37001818893228 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚36054727957504 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22172733 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8270097 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858111600150106e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚36137268 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚36054744996864 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1847434039808 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚37902161997312 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6018528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1847434039808 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6018764 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚165 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13954877 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14288861 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36127610 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1847521469440 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1842912343756 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚38038301885952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1842998146880 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚38038461658112 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚37902266466304 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚37902266466304 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚36140769614848 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚36140794518528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚35158906549472 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22182389 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1847521469440 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚37001921735712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22182389 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚236 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚36054744996864 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚36054727957504 trainer/global_step‚2099 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13954879 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22183256 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37920312856064 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14295367 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6021335 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚38056583620096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚37019436538770 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1843789816978 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1848313659392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚36071894720512 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 trainer/global_step‚2099 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8274034 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚35175646721792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1843875620214 _step‚166 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13961520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13961522 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1848313659392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22183256 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚36154448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1848401096192 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚36154448 % _timestamp‚1.7858111603676884e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22192926 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚38056421757952 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚36071894720512 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _runtime‚ 905.558027457 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37920208379904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37920312856064Gjìñ$…ÂM ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚36154448 % _timestamp‚1.7858111603676884e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22192926 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚38056421757952 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚36071894720512 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _runtime‚ 905.558027457 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37920208379904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37920312856064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22183256 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37920312856064 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14295367 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6021335 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚38056583620096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚37019436538770 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1843789816978 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1848313659392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚36071894720512 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 trainer/global_step‚2099 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8274034 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚35175646721792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1843875620214 _step‚166 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13961520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13961522 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1848313659392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22183256 K>DeviceStatsMonitor.on_train_batch_ewßÕ:³nd/allocation.all.allocated‚36154448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1848401096192 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121¦)Àq.0†+ _wandbruntime‚905  _runtime‚905XÃ’¦þ‡ø 2loss/SRqqt1/assignment_loss‚1.6759676897525788 .loss/FRt2/detection_loss‚3.051459579443445 0loss/SRqqt2/detection_loss‚3.732429415804066 .loss/FBt1/detection_loss‚9.300765919403284 &loss/total_loss‚38.180613262306316 trainer/global_step‚2099 _runtime‚ 905.558737524 2loss/SRqqt2/assignment_loss‚1.6237940416760641 0loss/FBt1/assignment_loss‚0.2153604492042704 /loss/FRt1/detection_loss‚3.2075786920154794 .loss/FBt2/detection_loss‚5.447610946217383 $ _timestamp‚1.785811160368938e+09 .loss/FRt1/assignment_loss‚4.38617604339824 /loss/FRt2/assignment_loss‚4.040206393417047 1loss/FBt2/assignment_loss‚0.14493004408897833 1loss/SRqqt1/detection_loss‚1.3543340478854777 epoch‚0 _step‚1671^.Έý .loss/FRt1/assignment_loss‚4.38617604339824 /loss/FRt2/assignment_loss‚4.040206393417047 1loss/FBt2/assignment_loss‚0.14493004408897833 1loss/SRqqt1/detection_loss‚1.3543340478854777 epoch‚0 _step‚167 2loss/SRqqt1/assignment_loss‚1.6759676897525788 .loss/FRt2/detection_loss‚3.051459579443445 0loss/SRqqt2/detection_loss‚3.732429415804066 .loss/FBt1/detection_loss‚9.300765919403284 &loss/total_loss‚38.180613262306316 trainer/global_step‚2099 _runtime‚ 905.558737524 2loss/SRqqt2/assignment_loss‚1.6237940416760641 0loss/FBt1/assignment_loss‚0.2153604492042704 /loss/FRt1/detection_loss‚3.2075786920154794 .loss/FBt2/detection_loss‚5.447610946217383 $ _timestamp‚1.785811160368938e+09§Û—b‰‚(:’ á¡ÅÓƒö‚ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚126.609' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚42.202999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0óø?VŠ‚(:Ë  á¡ÅÓäš¼ƒ disk.drbd1931.in‚0 disk.drbd1289.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0$ proc.memory.rssMB‚11254.03515625% memory_percent‚41.224437952041626 disk.rbd1.out‚ 3.6689453125* proc.memory.availableMB‚38519.15234375 disk.rbd1.in‚0 disk.drbd1584.out‚0 disk.sda.in‚ 0.41015625 disk.drbd1550.out‚0 cpu‚13.999871707473071) disk./.usagePercent‚32.79243813612224 disk.drbd1289.in‚0 disk.sda1.in‚0 disk.drbd1931.out‚0 disk.sda.out‚27.71875 disk.drbd1965.out‚0 network.recv‚178067 disk.drbd1828.in‚0 disk.rbd0.in‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.dm-0.in‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd1817.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd2111.in‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd2111.out‚0 disk.nvme0n1.in‚0 disk.drbd1550.in‚0 disk.drbd2078.out‚0 network.sent‚ 1.960442e+06 disk.drbd1552.out‚0 disk.drbd1356.out‚0 disk.drbd1552.in‚0 disk.drbd1817.in‚0 disk.sda2.in‚ 0.41015625 disk.sda2.out‚27.71875# disk.nvme0n1.out‚553.9697265625 disk.drbd2041.out‚0 disk.drbd1226.out‚0$ disk./.usageGB‚586.3620986938477 disk.drbd1828.out‚0 disk.dm-0.out‚553.9697265625 proc.cpu.threads‚29.t0‹+ _wandbruntime‚926  _runtime‚926d²jFÇŒÁ ' lr-AdamW/pg2‚5.5384428756922805e-05 trainer/global_step‚2149 $ _timestamp‚1.785811181506896e+09 _runtime‚ 926.696838355 _step‚168 ' lr-AdamW/pg1‚5.5384428756922805e-05µÃÌÆ ' lr-AdamW/pg1‚5.5384428756922805e-05 ' lr-AdamW/pg2‚5.5384428756922805e-05 trainer/global_step‚2149 $ _timestamp‚1.785811181506896e+09 _runtime‚ 926.696838355 _step‚168¨žTü0Ž+ _wandbruntime‚926  _runtime‚926Q³Lþ·'±O E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14287029 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚36913083147264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6162661 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _runtime‚ 926.701626881 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚38804496283136 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14287029 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚37001125454848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1891517604864 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚38943853768192 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚37882715972736 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚38804496283136 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8466947 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚38943998860288 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚36913066107904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14287027 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚37882818815220 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36987762 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1942848501760 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1886800807264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1891517604864 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37001150358528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚36997420 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚36913083147264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 145092096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚35995932204832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6162411 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14629356 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22710391 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1942728313344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚36913066107904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1886886610388 _step‚169 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚38804496283136 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8466947 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚38943998860288 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚36913066107904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14287027 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚37882818815220 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36987762 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1942848501760 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1886800807264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1891517604864 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37001150358528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚36997420 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚36913083147264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 145092096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚35995932204832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6162411 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14629356 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22710391 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1942728313344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚36913066107904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1886886610388 _step‚169 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1943631262720 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6165252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22711258 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 trainer/global_step‚2149 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚170 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1892397231616 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1892309794816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8470884 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚38822647141888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14293672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚36930249910272 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22720928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14636134 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1887678280486 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37004928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6165508 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37018357475328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚38822542665728 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚36930249910272 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37014600 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚37900333618278 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37004928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚38962119405056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 % _timestamp‚1.7858111818652263e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚36930232870912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22720928 b䱑 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37014600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚38822647141888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1892309794816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14293670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1892397231616 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8470882 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22711258 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚36930232870912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14636392ÙP“Ç&“ÁM _step‚170 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1892397231616 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1892309794816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8470884 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚38822647141888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14293672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚36930249910272 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22720928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14636134 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1887678280486 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37004928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6165508 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37018357475328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚38822542665728 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚36930249910272 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37014600 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚37900333618278 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37004928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚38962119405056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 % _timestamp‚1.7858111818652263e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚36930232870912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22720928 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37014600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚38822647141888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1892309794816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14293670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1892397231616 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8470882 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22711258 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚36930232870912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14636392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚36012655337792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14293672 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚37900436460874 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1943631262720 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6165252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22711258 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 trainer/global_step‚2149 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0ª% ä¿0”+ _wandbruntime‚927  _runtime‚927A4ôÓ•ü _step‚171 .loss/FRt1/detection_loss‚4.216484475135803 2loss/SRqqt2/assignment_loss‚1.6816287715751004 3loss/FBt2/assignment_loss‚0.0016502390426467173 0loss/SRqqt2/detection_loss‚3.174981802163354 _runtime‚ 927.056236114 /loss/FRt1/assignment_loss‚4.520506033530602 /loss/FRt2/assignment_loss‚3.648399700289187 .loss/FRt2/detection_loss‚3.106120544283286 .loss/FBt2/detection_loss‚5.136180388099979 %loss/total_loss‚40.58216388597414 % _timestamp‚1.7858111818664498e+09 2loss/SRqqt1/assignment_loss‚1.6437926998075385 1loss/FBt1/assignment_loss‚0.20596469072988838 1loss/SRqqt1/detection_loss‚1.6825315404041041 /loss/FBt1/detection_loss‚11.563923000912656 epoch‚0 trainer/global_step‚2149#ÊŒé– 1loss/FBt1/assignment_loss‚0.20596469072988838 1loss/SRqqt1/detection_loss‚1.6825315404041041 /loss/FBt1/detection_loss‚11.563923000912656 epoch‚0 trainer/global_step‚2149 _step‚171 .loss/FRt1/detection_loss‚4.216484475135803 2loss/SRqqt2/assignment_loss‚1.6816287715751004 3loss/FBt2/assignment_loss‚0.0016502390426467173 0loss/SRqqt2/detection_loss‚3.174981802163354 _runtime‚ 927.056236114 /loss/FRt1/assignment_loss‚4.520506033530602 /loss/FRt2/assignment_loss‚3.648399700289187 .loss/FRt2/detection_loss‚3.106120544283286 .loss/FBt2/detection_loss‚5.136180388099979 %loss/total_loss‚40.58216388597414 % _timestamp‚1.7858111818664498e+09 2loss/SRqqt1/assignment_loss‚1.6437926998075385«=Šëœ—‚(:‘ ð¡ÅÓ¡¤•ƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚97.423' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚32.474333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0okˆŠU˜‚(:Ê  ð¡ÅÓ£úàƒ disk.dm-0.in‚0 disk.drbd1938.out‚0 disk.drbd1931.out‚0 disk.drbd1552.out‚0 network.recv‚180519 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.drbd2118.out‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.41015625 network.sent‚ 2.005394e+06 disk.rbd1.out‚ 3.7314453125$ proc.memory.rssMB‚11254.03515625 cpu‚13.994704393678823 disk.rbd0.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0 proc.cpu.threads‚29 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd2078.in‚0 disk.dm-0.out‚ 554.7421875 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.drbd1965.out‚0 disk.drbd1285.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.sda.out‚ 28.32421875) disk./.usagePercent‚32.79246949678484 disk.drbd1285.in‚0* proc.memory.percent‚17.172294855117798 disk.sda1.in‚0$ disk./.usageGB‚586.3626594543457 disk.drbd1550.out‚0 disk.drbd1828.out‚0 disk.drbd1226.in‚0 disk.drbd2111.out‚0 disk.drbd1584.out‚0 disk.drbd1226.out‚0 disk.drbd1356.in‚0$ memory_percent‚41.22459292411804 disk.drbd1289.in‚0 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.nvme0n1.out‚ 554.7421875 disk.drbd2118.in‚0* proc.memory.availableMB‚38519.05078125 disk.sda2.in‚ 0.41015625 disk.sda2.out‚ 28.32421875 disk.drbd1356.out‚0 disk.drbd1584.in‚0IaS…™‚(:… ÿ¡ÅÓªžËƒ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚118.905' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚39.635 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0õ®jåPš‚(:Å  ÿ¡ÅÓ¶Ñ܃ disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.sda1.out‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.rbd1.out‚ 3.8251953125$ disk./.usageGB‚586.3625221252441 disk.drbd2041.in‚0 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.dm-0.out‚ 555.50390625 disk.sda1.in‚0 disk.drbd2118.in‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0$ memory_percent‚41.22421741485596 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.sda2.in‚ 0.41015625! disk.nvme0n1.out‚ 555.50390625 cpu‚13.988832165470967 proc.cpu.threads‚29) disk./.usagePercent‚32.79246181662258 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1584.in‚0 disk.drbd2041.out‚0 disk.drbd1817.in‚0* proc.memory.percent‚17.172294855117798 disk.drbd1289.in‚0 disk.sda.in‚ 0.41015625 disk.dm-0.in‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.drbd1285.in‚0 disk.drbd1938.in‚0 network.recv‚183009 disk.sda.out‚ 28.703125 disk.drbd1552.out‚0$ proc.memory.rssMB‚11254.03515625( proc.memory.availableMB‚ 38519.296875 disk.drbd1584.out‚0 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0 disk.sda2.out‚ 28.703125 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.nvme0n1.in‚0 network.sent‚ 2.05038e+06 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0¦W©U0›+ _wandbruntime‚947  _runtime‚947P݈’Äœ¾ % lr-AdamW/pg1‚5.66730380811881e-05 % lr-AdamW/pg2‚5.66730380811881e-05 trainer/global_step‚2199 % _timestamp‚1.7858112026865668e+09 _runtime‚ 947.876535487 _step‚172Wë˲Éà % lr-AdamW/pg1‚5.66730380811881e-05 % lr-AdamW/pg2‚5.66730380811881e-05 trainer/global_step‚2199 % _timestamp‚1.7858112026865668e+09 _runtime‚ 947.876535487 _step‚172¬wEB0ž+ _wandbruntime‚947  _runtime‚947dWž¸'Ÿ²O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚39706830568960 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14619177 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23238393 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6305897 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚39706935037952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1935426310656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8663797 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1988050531840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1935513740288 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23228737 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1930775073896 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37861506198528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1987915663360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚37771421297664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚38763715894728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚39706830568960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37857572 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14619179 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 trainer/global_step‚2199 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14619179 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚38763613052244 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚36832940820832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14969935 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14619177 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1935513740288 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚37771421297664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚37861481294848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23228737 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚39706935037952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚36832923781472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚ 947.880637194 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2v@•× ·O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1935513740288 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚37771421297664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚37861481294848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23228737 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚39706935037952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚36832923781472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚ 947.880637194 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚39706830568960 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14619177 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23238393 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6305897 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚39706935037952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1935426310656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8663797 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1988050531840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1935513740288 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23228737 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1930775073896 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37861506198528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1987915663360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚37771421297664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚38763715894728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚39706830568960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37857572 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14619179 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 trainer/global_step‚2199 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14619179 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚38763613052244 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚36832940820832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14969935 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14619177 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121­¦f?Ê0¡+ _wandbruntime‚948  _runtime‚948UŒd8Ã&¢½M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14625820 trainer/global_step‚2199 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1936393367040 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37874752 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37874752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚37878688411648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚39867668739072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚37788588060672 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6308962 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1931652547230 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚174 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14625822 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1936393367040 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚39867500585472 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37865080 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14976450 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14625820 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 _runtime‚ 948.229858631 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8667734 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚38781333540382 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23248930 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚37788571021312 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1988812173824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0”Ü¡È&£ÂM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14625820 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 _runtime‚ 948.229858631 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8667734 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚38781333540382 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23248930 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚37788571021312 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1988812173824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14625820 trainer/global_step‚2199 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1936393367040 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37874752 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37874752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚37878688411648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚39867668739072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚37788588060672 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6308962 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1931652547230 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚174 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14625822 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1936393367040 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚39867500585472 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37865080 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14976450 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251®FÓÝ0¤+ _wandbruntime‚948  _runtime‚948™lb­¥û /loss/FRt1/assignment_loss‚4.464280733248082 2loss/SRqqt1/assignment_loss‚1.5531789709267945 2loss/SRqqt2/assignment_loss‚1.6399091316174856 0loss/FBt1/assignment_loss‚0.4064322412923502 3loss/FBt2/assignment_loss‚0.0059105744658154435 1loss/SRqqt1/detection_loss‚1.5350826789903866 1loss/SRqqt2/detection_loss‚3.2645242395280283 $ _timestamp‚1.785811203040776e+09 /loss/FRt2/assignment_loss‚3.670461371138289 .loss/FBt1/detection_loss‚20.46359684939186 .loss/FBt2/detection_loss‚8.706068583253131 %loss/total_loss‚52.78082211273909 _runtime‚ 948.230704762 .loss/FRt2/detection_loss‚3.800768619856319 epoch‚0 _step‚175 /loss/FRt1/detection_loss‚3.2706081190305514 trainer/global_step‚2199¨s¦€ trainer/global_step‚2199 /loss/FRt1/assignment_loss‚4.464280733248082 2loss/SRqqt1/assignment_loss‚1.5531789709267945 2loss/SRqqt2/assignment_loss‚1.6399091316174856 0loss/FBt1/assignment_loss‚0.4064322412923502 3loss/FBt2/assignment_loss‚0.0059105744658154435 1loss/SRqqt1/detection_loss‚1.5350826789903866 1loss/SRqqt2/detection_loss‚3.2645242395280283 $ _timestamp‚1.785811203040776e+09 /loss/FRt2/assignment_loss‚3.670461371138289 .loss/FBt1/detection_loss‚20.46359684939186 .loss/FBt2/detection_loss‚8.706068583253131 %loss/total_loss‚52.78082211273909 _runtime‚ 948.230704762 .loss/FRt2/detection_loss‚3.800768619856319 epoch‚0 _step‚175 /loss/FRt1/detection_loss‚3.2706081190305514¯†ðý§‚(:’ Ž¢ÅÓãñƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚100.176' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚33.391999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ùZ¨‚(:Ï  Ž¢ÅÓ̳փ cpu‚13.983865655587524 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 3.9736328125 disk.drbd1356.in‚0 disk.drbd1965.out‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.nvme0n1.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1938.out‚0 proc.cpu.threads‚29 disk.drbd1931.in‚0 disk.sda.out‚ 29.94921875* disk./.usagePercent‚32.792465870041546 disk.rbd0.in‚0 disk.drbd1510.in‚0 disk.drbd1817.out‚0$ disk.nvme0n1.out‚556.50830078125 disk.drbd1584.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1356.out‚0& proc.memory.availableMB‚ 38518.9375* proc.memory.percent‚17.172294855117798 disk.drbd1550.in‚0 disk.drbd1817.in‚0 disk.sda2.out‚ 29.94921875 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1673.in‚0 disk.drbd1584.out‚0$ memory_percent‚41.22476577758789$ proc.memory.rssMB‚11254.03515625 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1938.in‚0$ disk./.usageGB‚586.3625946044922! disk.dm-0.out‚556.50830078125 disk.sda1.out‚0 disk.drbd1931.out‚0 disk.sda.in‚ 0.41015625 disk.drbd1226.in‚0 disk.drbd1828.out‚0 network.recv‚183970 disk.drbd1289.out‚0 disk.drbd2041.out‚0 disk.drbd1226.out‚0 network.sent‚ 2.053421e+06 disk.drbd2118.in‚0 disk.drbd1965.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0 disk.drbd1552.out‚0 disk.sda1.in‚0 disk.drbd2118.out‚0 disk.drbd2041.in‚0Ì•o0©+ _wandbruntime‚969  _runtime‚969ŽxѱȪ ' lr-AdamW/pg1‚5.7961647405453414e-05 ' lr-AdamW/pg2‚5.7961647405453414e-05 trainer/global_step‚2249 % _timestamp‚1.7858112239720364e+09 _runtime‚ 969.161979928 _step‚176äâ9ŠÍ«Ç trainer/global_step‚2249 % _timestamp‚1.7858112239720364e+09 _runtime‚ 969.161979928 _step‚176 ' lr-AdamW/pg1‚5.7961647405453414e-05 ' lr-AdamW/pg2‚5.7961647405453414e-05°|µäx0¬+ _wandbruntime‚969  _runtime‚969‹œÖn“ ­³O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚38629742408704 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1979509875712 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23766395 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15310648 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚39644510131752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23756739 HBDeviceStatsMonitor.on_train_batch_start%~¥†&/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚38708066 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚40609269323776 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23756739 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚38629759448064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15310409 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1979422446080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23766395 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚38721862038528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚38629742408704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1979422446080 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40755062489600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚37669932397472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1974577734280 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚2249 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14951327 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2033069776896 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚39644612974236 _runtime‚ 969.166625415 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚40609164854784 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚38721837134848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858112239729557e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚38717724 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8860645 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6450001 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚40754906911744 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14951327 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚40609164854784 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8860647 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚40609269323776 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14951329 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚177 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1979509875712 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0š¶¾'®¸O =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14951329 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚177 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1979509875712 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚38629742408704 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1979509875712 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23766395 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15310648 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚39644510131752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23756739 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚38708066 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚40609269323776 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23756739 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚38629759448064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15310409 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1979422446080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23766395 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚38721862038528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚38629742408704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1979422446080 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40755062489600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚37669932397472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1974577734280 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚2249 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14951327 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2033069776896 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚39644612974236 _runtime‚ 969.166625415 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚40609164854784 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚38721837134848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858112239729557e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚38717724 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8860645 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6450001 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚40754906911744 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14951327 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚40609164854784 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8860647 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚40609269323776±Ó¨oÃ0¯+ _wandbruntime‚969  _runtime‚969Çë€Ã&°½M UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚37686689609152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38734904 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2034105809920 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚37686672569792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23776932 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2033968851456 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1975541010738 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚38646926211072 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚38739044251648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚178 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38734904 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1980302065664 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚40773013103104 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚39662127777294 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14957972 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8864582 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1980389502464 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚38646926211072 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23767262 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚274 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1975455207502 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚38646909171712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15317459 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚38646909171712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14957972 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15317183 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1980389502464 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23776932 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚276 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858112243245718e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14957970 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚38739069155328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚38725232 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚40627315713536 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251·L.±ÂM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14957970 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚38739069155328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚38725232 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚40627315713536 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚37686689609152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38734904 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2034105809920 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚37686672569792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23776932 ODeviceStatsMonitor.on_train_batch_end/allocated_b‹¯?Âytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2033968851456 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1975541010738 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚38646926211072 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚38739044251648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚178 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38734904 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1980302065664 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚40773013103104 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚39662127777294 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14957972 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8864582 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1980389502464 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚38646926211072 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23767262 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚274 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1975455207502 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚38646909171712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15317459 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚38646909171712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14957972 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15317183 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1980389502464 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23776932 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚276 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858112243245718e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160²ÜvŒ0²+ _wandbruntime‚969  _runtime‚969?»³ú .loss/FRt1/detection_loss‚3.669751147905441 0loss/SRqqt2/detection_loss‚3.443151182555533 &loss/total_loss‚35.091765495525046 _runtime‚ 969.515556404 /loss/FRt2/assignment_loss‚3.787030451827579 2loss/SRqqt2/assignment_loss‚1.6620908929156017 1loss/FBt2/assignment_loss‚0.05317366295639658 /loss/FRt2/detection_loss‚3.1797049654854668 epoch‚0 trainer/global_step‚2249 $ _timestamp‚1.785811224325781e+09 /loss/FRt1/assignment_loss‚4.547841761210194 1loss/FBt1/assignment_loss‚0.23494291667686246 1loss/SRqqt1/detection_loss‚1.2416884723909225 2loss/SRqqt1/assignment_loss‚1.5762683372944593 .loss/FBt1/detection_loss‚7.382569307630712 .loss/FBt2/detection_loss‚4.313552396675878 _step‚179ÜTV´ÿ /loss/FRt1/assignment_loss‚4.547841761210194 1loss/FBt1/assignment_loss‚0.23494291667686246 1loss/SRqqt1/detection_loss‚1.2416884723909225 2loss/SRqqt1/assignment_loss‚1.5762683372944593 .loss/FBt1/detection_loss‚7.382569307630712 .loss/FBt2/detection_loss‚4.313552396675878 _step‚179 .loss/FRt1/detection_loss‚3.669751147905441 0loss/SRqqt2/detection_loss‚3.443151182555533 &loss/total_loss‚35.091765495525046 _runtime‚ 969.515556404 /loss/FRt2/assignment_loss‚3.787030451827579 2loss/SRqqt2/assignment_loss‚1.6620908929156017 1loss/FBt2/assignment_loss‚0.05317366295639658 /loss/FRt2/detection_loss‚3.1797049654854668 epoch‚0 trainer/global_step‚2249 $ _timestamp‚1.785811224325781e+09³§°‘\µ‚(:Ñ  ¢ÅÓÊŽ¼ƒ disk.drbd1550.out‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 cpu‚13.979287905851635 disk.drbd1356.in‚0 disk.drbd1550.in‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0$ disk.nvme0n1.out‚561.97607421875 disk.drbd1584.out‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 disk.drbd1552.in‚0 network.sent‚ 2.098341e+06 disk.sda.out‚ 31.6015625 disk.rbd0.out‚0 disk.drbd1285.out‚0 proc.cpu.threads‚29 disk.drbd1584.in‚0 disk.drbd1931.out‚0$ proc.memory.rssMB‚11254.03515625 disk.sda2.in‚ 0.41015625 disk.sda2.out‚ 31.6015625 disk.rbd1.out‚ 3.9736328125 disk.sda.in‚ 0.41015625 disk.drbd2078.in‚0! disk.dm-0.out‚561.97607421875 disk.drbd2111.out‚0 disk.drbd1828.in‚0 disk.drbd2041.in‚0 disk.drbd1285.in‚0 disk.drbd1673.out‚0) proc.memory.availableMB‚ 38519.0859375 network.recv‚186183$ disk./.usageGB‚586.3625602722168 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd1226.in‚0 disk.drbd1510.out‚0 disk.sda1.in‚0* proc.memory.percent‚17.172294855117798 disk.nvme0n1.in‚0 disk.drbd1226.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0% memory_percent‚41.224539279937744 disk.drbd1965.in‚0* disk./.usagePercent‚32.792463950000986 disk.drbd1938.out‚0 disk.drbd1931.in‚0 disk.dm-0.in‚0 disk.drbd1673.in‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd2118.in‚0˜à­*œ¶‚(:‘ ¢ÅÓ•¹¶ƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚95.983' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚31.994333333333337 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0æì«Nœ·‚(:‘ ¬¢ÅÓ»šä‚ gpu.0.gpu‚39.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚113.72' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.906666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01Jb¸‚(:×  ¬¢ÅӃ骃 disk.drbd1673.out‚0 disk.drbd2078.out‚0 network.sent‚ 2.143308e+06# disk.nvme0n1.out‚565.9091796875 disk.drbd2118.out‚0 disk.dm-0.out‚565.9091796875$ proc.memory.rssMB‚11254.03515625 proc.cpu.threads‚29 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.drbd1226.in‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.nvme0n1.in‚2.46875 disk.rbd0.in‚0 disk.drbd1285.in‚0) disk./.usagePercent‚32.79244986970349 disk.dm-0.in‚2.46875 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd1356.in‚0 disk.drbd1584.out‚0 disk.sda1.in‚0 disk.drbd2041.out‚0 disk.sda.in‚ 0.41015625 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.drbd1550.in‚0 disk.rbd1.in‚0 disk.drbd2078.in‚0 cpu‚13.974957348781029 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1965.out‚0$ disk./.usageGB‚586.3623085021973 disk.rbd1.out‚ 4.0361328125 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.sda.out‚ 32.8203125 disk.rbd0.out‚0# memory_percent‚41.2245512008667( proc.memory.availableMB‚ 38519.078125 disk.drbd1965.in‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1828.out‚0 disk.sda2.out‚ 32.8203125 disk.drbd1289.out‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 network.recv‚188665 disk.drbd1817.out‚0 disk.drbd1673.in‚0* proc.memory.percent‚17.172294855117798 disk.drbd2111.out‚0Nuµ0¹+ _wandbruntime‚990  _runtime‚9902îȺ % _timestamp‚1.7858112450777495e+09 _runtime‚ 990.267694005 _step‚180 ' lr-AdamW/pg1‚5.9250256729718725e-05 ' lr-AdamW/pg2‚5.9250256729718725e-05 trainer/global_step‚2299F£1@Í»Ç ' lr-AdamW/pg1‚5.9250256729718725e-05 ' lr-AdamW/pg2‚5.9250256729718725e-05 trainer/global_step‚2299 % _timestamp‚1.7858112450777495e+09 _runtime‚ 990.267694005 _step‚180´þUY0¼+ _wandbruntime‚990  _runtime‚990h¶¼“¹'½³O _runtime‚ 990.272640184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39577876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6594300 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚39488080559104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15283477 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24284741 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚39488080559104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15283479 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _step‚181 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39568218 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚41511603609600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2018466197788 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9057497 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24294397 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚41511499140608 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚41660632316928 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15652019 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚40525510053744 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚41511499140608 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚38506941013472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚39488097598464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚41660472544768 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15283477 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24294397 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2078414438400 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚224 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚222 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚39582217878528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚41511603609600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚41511603609600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2018466197788 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9057497 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24294397 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚41511499140608 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚41660632316928 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15652019 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚40525510053744 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚41511499140608 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚38506941013472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚39488097598464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚41660472544768 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15283477 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24294397 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2078414438400 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚224 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚222 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚39582217878528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚41511603609600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _step‚181 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39568218µQHÒ¾0¿+ _wandbruntime‚990  _runtime‚990^ÐmEÀ½M G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39595056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9061434 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 trainer/global_step‚2299 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejec¶n‰¼"tions‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚38523698225152 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2024298201088 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15290122 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2024298201088 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6597361 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24295264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6597115 _runtime‚ 990.618142611 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24304934 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15658795 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚38523681185792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2019429474246 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚41678603230208 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15658547 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39585384 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚39505264361472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚39505247322112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚41529649999360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 _step‚182 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚41529545523200 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚40543024856802 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39595056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚40543127699398 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚41678754606592 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6597115 _runtime‚ 990.618142611 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24304934 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15658795 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚38523681185792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2019429474246 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚41678603230208 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15658547 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39585384 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚39505264361472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚39505247322112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚41529649999360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 _step‚182 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚41529545523200 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚40543024856802 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39595056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚40543127699398 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚41678754606592 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚38523698225152 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2024298201088 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15290122 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2024298201088 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6597361 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24295264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚39343966668832 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚40428370 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42413833426432 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2067414716928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2067502146560 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15615627 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2123707158016 trainer/global_step‚2349 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15990740 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2123580678144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚40346435748864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15615629 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚40442548814848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚40438028 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15615629 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42413937895424 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚42413937895424 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚42566280876544 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24812743 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2067502146560 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24822399 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6736393 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚42566129492992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2062440464420 ?9DeviceStatsMonitor.KÖlϾon_train_batch_start/segment.all.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9254347 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6736156 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚04>‰Å¿'̹O MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15990740 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2123580678144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚40346435748864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15615629 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚40442548814848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚40438028 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15615629 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42413937895424 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚42413937895424 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚42566280876544 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24812743 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2067502146560 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24822399 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6736393 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚42566129492992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2062440464420 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9254347 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6736156 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24822399 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15615627 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15990501 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚40442573718528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚40346418709504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚185 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚39343966668832 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚40428370 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42413833426432 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2067414716928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2067502146560 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15615627 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2123707158016 trainer/global_step‚2349 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640¹‹^º42Í- _wandbruntime‚1011  _runtime‚1011ÈŒÃ&νM _step‚186 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚42431879809024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚40363602511872 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2068294336512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚41423921936310 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2063317937754 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40455208 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24823266 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24832936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40459755931648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚40363602511872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15622272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚40445536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚40363585472512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2068381773312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9258282 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15622270 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6739005 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚1011.562948928 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15622270 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2063232134518 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚42584401978368 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15997287 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚42584244310528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚40363585472512 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2124488378880 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 trainer/global_step‚2349 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚41424024778906 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6739258 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2068381773312 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42431984285184 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2124621143040 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40455208 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 $ _timestamp‚1.785811266372671e+09 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42431984285184 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚39360689801792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15622272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9258284 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24832936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2068294336512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40459780835328 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24823266 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15997542 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538O™lÈ&ÏÂM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24823266 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24832936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40459755931648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚40363602511872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15622272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚40445536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚40363585472512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2068381773312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9258282 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15622270 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6739005 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚1011.562948928 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15622270 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2063232134518 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚42584401978368 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15997287 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚42584244310528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚40363585472512 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2124488378880 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 trainer/global_step‚2349 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚41424024778906 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6739258 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2068381773312 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42431984285184 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2124621143040 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40455208 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 $ _timestamp‚1.785811266372671e+09 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42431984285184 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚39360689801792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15622272 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9258284 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24832936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2068294336512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40459780835328 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24823266 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15997542 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _step‚186 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚42431879809024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚40363602511872 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2068294336512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚41423921936310 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2063317937754 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40455208ºtvIÒ2Ð- _wandbruntime‚1011  _runtime‚1011GUÿÑý /loss/FRt2/assignment_loss‚3.904943440641676 2loss/SRqqt2/assignment_loss‚1.7125418826061136 1loss/FBt1/assignment_loss‚0.39524328969524836 1loss/FBt2/assignment_loss‚0.03630032224398085 .loss/FRt1/detection_loss‚3.371759223280015 .loss/FBt1/detection_loss‚17.40353326473004 trainer/global_step‚2349 _runtime‚1011.563676366 /loss/FRt2/detection_loss‚3.2828125566953705 0loss/SRqqt1/detection_loss‚1.236570845799577 /loss/FBt2/detection_loss‚3.9747607569243595 &loss/total_loss‚44.126758070962026 _step‚187 2loss/SRqqt1/assignment_loss‚1.6119086458658178 /loss/FRt1/assignment_loss‚4.340402346152763 1loss/SRqqt2/detection_loss‚2.8559814963270638 epoch‚0 % _timestamp‚1.7858112663738723e+09SI!ôÒ‚ 1loss/SRqqt2/detection_loss‚2.8559814963270638 epoch‚0 % _timestamp‚1.7858112663738723e+09 /loss/FRt2/assignment_loss‚3.904943440641676 2loss/SRqqt2/assignment_loss‚1.7125418826061136 1loss/FBt1/assignment_loss‚0.39524328969524836 1loss/FBt2/assignment_loss‚0.03630032224398085 .loss/FRt1/detection_loss‚3.371759223280015 .loss/FBt1/detection_loss‚17.40353326473004 trainer/global_step‚2349 _runtime‚1011.563676366 /loss/FRt2/detection_loss‚3.2828125566953705 0loss/SRqqt1/detection_loss‚1.236570845799577 /loss/FBt2/detection_loss‚3.9747607569243595 &loss/total_loss‚44.126758070962026 _step‚187 2loss/SRqqt1/assignment_loss‚1.6119086458658178 /loss/FRt1/assignment_loss‚4.340402346152763»8£z·œÓ‚(:‘ Ê¢ÅÓÌéƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚128.963' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.98766666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0é|§Ô‚(:Ú  Ê¢ÅÓÜÀÒƒ disk.drbd2118.in‚0 disk.drbd1965.in‚0 disk.drbd2041.out‚0 disk.sda.out‚ 33.51953125 disk.drbd1226.in‚0 disk.drbd21zÙ¸S¾18.out‚0 disk.drbd1931.out‚0 disk.dm-0.in‚2.46875 proc.cpu.threads‚29 network.recv‚192238 disk.sda2.in‚ 0.41015625 disk.drbd1552.out‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.sda2.out‚ 33.51953125" disk.nvme0n1.out‚ 566.736328125 disk.sda.in‚ 0.41015625 disk.drbd1510.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1356.in‚0 disk.drbd1584.out‚0 cpu‚13.966409605534121 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0$ proc.memory.rssMB‚11254.03515625 network.sent‚ 2.191417e+06 disk.drbd1552.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1931.in‚0 disk.dm-0.out‚ 566.736328125 disk.drbd1673.in‚0 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0) disk./.usagePercent‚32.79244986970349 disk.rbd1.out‚ 4.1923828125* proc.memory.percent‚17.172294855117798$ memory_percent‚41.22430682182312* proc.memory.availableMB‚38519.23828125 disk.drbd2078.out‚0$ disk./.usageGB‚586.3623085021973 disk.drbd1817.in‚0 disk.drbd1550.in‚0 disk.drbd1938.in‚0g°í2Õ- _wandbruntime‚1032  _runtime‚1032Dú`NÇÖÁ % _timestamp‚1.7858112871590648e+09 _runtime‚1032.349004285 _step‚188 & lr-AdamW/pg1‚6.182747537824933e-05 & lr-AdamW/pg2‚6.182747537824933e-05 trainer/global_step‚2399Ñ°ú_Ì×Æ _step‚188 & lr-AdamW/pg1‚6.182747537824933e-05 & lr-AdamW/pg2‚6.182747537824933e-05 trainer/global_step‚2399 % _timestamp‚1.7858112871590648e+09 _runtime‚1032.349004285¼ådü“2Ø- _wandbruntime‚1032  _runtime‚1032uÖÔº'Ù´O ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2111498281984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2168783853056 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚41204773899264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚41204756859904 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9451195 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6880749 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2168914527232 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15947779 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _runtime‚1032.353640146 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2111410852352 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚42287201370276 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15947777 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25350401 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚42287304212760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚40180975284832 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15947779 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25350401 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15947777 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16331946 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2111498281984 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16331689 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41288522 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43471688507904 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚2399 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚43316272181248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7858112871600049e+09 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚41204756859904 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚43316167712256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25340745 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚41302904654848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚43316167712256 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚189 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25340745£øur¿'Ú¹O F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚41204773899264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚41204756859904 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9451195 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6880749 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2168914527232 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15947779 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _runtime‚1032.353640146 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2111410852352 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚42287201370276 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15947777 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25350401 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚42287304212760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚40180975284832 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15947779 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25350401 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15947777 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16331946 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2111498281984 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16331689 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41288522 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43471688507904 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚2399 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚43316272181248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7858112871600049e+09 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚41204756859904 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚43316167712256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25340745 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚41302904654848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚43316167712256 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚189 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25340745 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2111498281984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2168783853056 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484½‘eÙ2Û- _wandbruntime‚1032  _runtime‚1032)r¹ªÃ&ܽM QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2112377908736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2169682788352 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15954422 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25360938 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚43489952227840 _runtime‚1032.701960998 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚43334214094848 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚41221940662272 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚41221923622912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 trainer/global_step‚2399 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9455132 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚190 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 $ _timestamp‚1.785811287511676e+09 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16338502 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6883619 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2112377908736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚43489794560000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚40197715457152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚40197698417792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2107206401262 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6883370 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2112290471936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15954420 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25360938 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚42304921858414 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15954420 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚41320136675328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚43334318571008 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41315360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25351268 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚41221923622912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 Bc—ÝÂM OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 A9DeviceStatsMonitor.on_tra­ƒŸ 1%in_batch_end/segment.all.current‚251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚43334318571008 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41315360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25351268 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚41221923622912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2112377908736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2169682788352 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15954422 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25360938 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚43489952227840 _runtime‚1032.701960998 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚43334214094848 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚41221940662272 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚41221923622912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 trainer/global_step‚2399 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9455132 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚190 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 $ _timestamp‚1.785811287511676e+09 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16338502 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6883619 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2112377908736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚43489794560000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚40197715457152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚40197698417792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2107206401262 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6883370 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2112290471936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15954420 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25360938 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚42304921858414 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15954420 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚41320136675328¾ ŠÎ2Þ- _wandbruntime‚1032  _runtime‚1032rh®ß€ .loss/FBt2/detection_loss‚4.268368856851742 epoch‚0 0loss/FRt1/assignment_loss‚4.3904974796402625 2loss/SRqqt2/assignment_loss‚1.8731229042119169 1loss/FBt1/assignment_loss‚0.10024648562830407 /loss/FRt1/detection_loss‚3.5361027121543884 /loss/FRt2/detection_loss‚3.0227569813256734 1loss/SRqqt1/detection_loss‚1.3987379025338638 2loss/FBt2/assignment_loss‚0.011905334353672735 _step‚191 0loss/FRt2/assignment_loss‚3.7410546449514537 2loss/SRqqt1/assignment_loss‚1.5729299174854532 %loss/total_loss‚35.71798866093488 trainer/global_step‚2399 % _timestamp‚1.7858112875129702e+09 _runtime‚1032.703074278 1loss/SRqqt2/detection_loss‚2.6473359603514064 .loss/FBt1/detection_loss‚9.154929481446743¾pè à… 0loss/FRt2/assignment_loss‚3.7410546449514537 2loss/SRqqt1/assignment_loss‚1.5729299174854532 %loss/total_loss‚35.71798866093488 trainer/global_step‚2399 % _timestamp‚1.7858112875129702e+09 _runtime‚1032.703074278 1loss/SRqqt2/detection_loss‚2.6473359603514064 .loss/FBt1/detection_loss‚9.154929481446743 .loss/FBt2/detection_loss‚4.268368856851742 epoch‚0 0loss/FRt1/assignment_loss‚4.3904974796402625 2loss/SRqqt2/assignment_loss‚1.8731229042119169 1loss/FBt1/assignment_loss‚0.10024648562830407 /loss/FRt1/detection_loss‚3.5361027121543884 /loss/FRt2/detection_loss‚3.0227569813256734 1loss/SRqqt1/detection_loss‚1.3987379025338638 2loss/FBt2/assignment_loss‚0.011905334353672735 _step‚191¿ÿו¨á‚(:„ Ù¢ÅÓ—Õ«ƒ gpu.0.gpu‚30.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚123.42' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚41.14 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0G÷dâ‚(:Ù  Ù¢ÅÓ‹¶ìƒ* proc.memory.availableMB‚38518.91796875 disk.drbd1550.out‚0 disk.drbd1289.in‚0* proc.memory.percent‚17.172294855117798 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.dm-0.out‚ 566.904296875 network.sent‚ 2.23636e+06 disk.drbd1938.out‚0" disk.nvme0n1.out‚ 566.904296875 disk.drbd1510.in‚0 disk.drbd1965.out‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd2111.out‚0 disk.drbd1584.in‚0 disk.rbd1.in‚0$ memory_percent‚41.22479557991028 disk.sda2.in‚ 0.41015625 disk.sda2.out‚ 33.81640625 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0 disk.rbd1.out‚ 4.1923828125 disk.drbd1931.out‚0 disk.drbd1550.in‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1356.in‚0 disk.drbd1285.in‚0 disk.dm-0.in‚2.46875 disk.drbd2041.in‚0$ disk./.usageGB‚586.3623085021973 disk.drbd2078.in‚0 disk.drbd1510.out‚0 proc.cpu.threads‚29 network.recv‚194719 disk.drbd1817.in‚0 disk.nvme0n1.in‚2.46875 disk.sda.out‚ 33.81640625 disk.drbd2118.out‚0 disk.sda.in‚ 0.41015625 disk.drbd1938.in‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 cpu‚13.961968059725553) disk./.usagePercent‚32.79244986970349 disk.drbd1673.in‚0 disk.drbd1356.out‚0fIöœã‚(:‘ è¢ÅÓ…ãò‚ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚120.575' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.19166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0òT~gä‚(:Ü  è¢ÅӜҷƒ disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.drbd2041.out‚0) proc.memory.availableMB‚ 38518.6640625 disk.drbd1510.out‚0 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.drbd2078.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1584.out‚0) disk./.usagePercent‚32.79245008304133 disk.nvme0n1.in‚2.46875 disk.sda.out‚ 34.24609375 disk.drbd1965.out‚0 disk.rbd0.out‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd1931.out‚0 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd1289.in‚0 disk.sda1.in‚0 disk.drbd1673.out‚0 disk.drbd1931.in‚0 network.recv‚197102 disk.dm-0.in‚2.46875 disk.rbd1.out‚ 4.224609375 disk.drbd1356.out‚0 disk.drbd1584.in‚0 disk.drbd2111.in‚0 disk.drbd2041.in‚0 disk.drbd2078.in‚0 cpu‚13.958673278182882$ disk./.usageGB‚586.3623123168945! disk.dm-0.out‚567.60302734375 disk.drbd1226.out‚0* proc.memory.percent‚17.172294855117798 network.sent‚ 2.281536e+06 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd1552.in‚0 disk.sda2.out‚ 34.24609375 disk.sda.in‚ 0.41015625 proc.cpu.threads‚29 disk.drbd1828.out‚0 disk.drbd1550.out‚0$ memory_percent‚41.22518301010132 disk.drbd2111.out‚0 disk.drbd1673.in‚0 disk.sda2.in‚ 0.41015625$ disk.nvme0n1.out‚567.603027343751[-2å- _wandbruntime‚1053  _runtime‚1053®ÀÖÇæÁ & lr-AdamW/pg2‚6.311608470251464e-05 trainer/global_step‚2449 % _timestamp‚1.7858113081070445e+09 _runtime‚1053.296989879 _step‚192 & lr-AdamW/pg1‚6.311608470251464e-05b:ÔÌçÆ _step‚192 & lr-AdamW/pg1‚6.311608470251464e-05 & lr-AdamW/pg2‚6.311608470251464e-05 trainer/global_step‚2449 % _timestamp‚1.7858113081070445e+09 _runtime‚1053.296989879À³¯JS2è- _wandbruntime‚1053  _runtime‚1053ÕÌ2¹'é³O ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2155406987776 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚42148674 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2155406987776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9648047 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚42063095010304 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25868747 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9648045 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚42163285398528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16672185 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚43168201292268 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚44218606467072 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16671937 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 $ _timestamp‚1.785811308107982e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16279929 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2155494417408 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7023892 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16279927 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚246 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚44218501998080 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚44377249830912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25878403 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2155494417408 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚246 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚44218501998080 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚44377249830912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25878403 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2155494417408 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚42148674 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2155406987776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9648047 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚42063095010304 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25868747 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9648045 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚42163285398528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16672185 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚43168201292268 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚44218606467072 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16671937 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032ÁÇØÓ2ë- _wandbruntime‚1053  _runtime‚1053è‘=~Ä&ì¾M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚43185716095326 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚194 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚42080261773312 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚44236548380672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16286572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25879270 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2151009061534 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25888940 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚42080278812672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9651982 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7026745 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚43185818937922 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2156374044160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚42080261773312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16286570 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚44236652856832 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2156286607360 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚44395358924288 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2151094864770 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16286570 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7027004 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2214891312640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚42180467611648 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42175512 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16678988 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16678727 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42165840 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42175512 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚42165840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7858113084707534e+09 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16286572 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚44395518689280 trainer/global_step‚2449 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25879270 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42080278812672 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚42180492515328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _runtime‚1053.661088306mëÉ&íÃM XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _runtime‚1053.661088306 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚43185716095326 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚194 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚42080261773312 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚44236548380672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16286572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25879270 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2151009061534 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25888940 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚42080278812672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9651982 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7026745 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚43185818937922 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2156374044160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚42080261773312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16286570 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚44236652856832 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2156286607360 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚44395358924288 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2151094864770 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16286570 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7027004 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2214891312640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚42180467611648 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42175512 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16678988 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16678727 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42165840 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42175512 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚42165840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7858113084707534e+09 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16286572 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚44395518689280 trainer/global_step‚2449 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25879270 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42080278812672 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚42180492515328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392Â[AxÆ2î- _wandbruntime‚1053  _runtime‚1053·êoôïú .loss/FRt1/assignment_loss‚4.06981655726066 1loss/FBt1/assignment_loss‚0.33411411162387594 1loss/SRqqt1/detection_loss‚1.2090677543424404 1loss/SRqqt2/detection_loss‚3.3972172222926584 -loss/FBt2/detection_loss‚5.67853059646637 % _timestamp‚1.7858113084719882e+09 0loss/FRt2/assignment_loss‚3.8671282173432027 1loss/SRqqt2/assignment_loss‚1.756629145481217 epoch‚0 _step‚195 2loss/SRqqt1/assignment_loss‚1.5834679799521063 0loss/FBt2/assignment_loss‚0.1079972984342021 /loss/FRt1/detection_loss‚3.2580292717768597 /loss/FRt2/detection_loss‚2.8232734006704745 .loss/FBt1/detection_loss‚7.213029882172355 %loss/total_loss‚35.29830143781642 trainer/global_step‚2449 _runtime‚1053.661779236qƒ3‹ðÿ 0loss/FRt2/assignment_loss‚3.8671282173432027 1loss/SRqqt2/assignment_loss‚1.756629145481217 epoch‚0 _step‚195 2loss/SRqqt1/assignment_loss‚1.5834679799521063 0loss/FBt2/assignment_loss‚0.1079972984342021 /loss/FRt1/detection_loss‚3.2580292717768597 /loss/FRt2/detection_loss‚2.8232734006704745 .loss/FBt1/detection_loss‚7.213029882172355 %loss/total_loss‚35.29830143781642 trainer/global_step‚2449 _runtime‚1053.661779236 .loss/FRt1/assignment_loss‚4.06981655726066 1loss/FBt1/assignment_loss‚0.33411411162387594 1loss/SRqqt1/detection_loss‚1.2090677543424404 1loss/SRqqt2/detection_loss‚3.3972172222926584 -loss/FBt2/detection_loss‚5.67853059646637 % _timestamp‚1.7858113084719882e+09ÃaI3Û‘ñ‚(:† ÷¢ÅÓÉ÷ï‚ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚131.052' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.684 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0mÝü®iò‚(:Þ  ÷¢ÅÓ€¤ºƒ) disk./.usagePercent‚32.79245008304133 disk.sda.out‚ 34.58203125 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0* proc.memory.availableMB‚38518.86328125 disk.sda2.out‚ 34.58203125 disk.rbd1.in‚0 disk.rbd1.out‚ 4.404296875 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1817.out‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.in‚0 disk.sda.in‚ 0.41015625 disk.drbd2118.out‚0 disk.drbd1226.in‚0 cpu‚13.954014058898057 disk.drbd2118.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1965.out‚0 disk.drbd1285.out‚0! disk.dm-0.out‚567.76318359375 disk.drbd1289.out‚0 disk.drbd2078.out‚0 disk.drbd1289.in‚0 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.drbd1510.in‚0 disk.drbd1552.in‚0 proc.cpu.threads‚29 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.drbd1226.out‚0 disk.drbd1510.out‚0 disk.sda1.in‚0 disk.drbd2041.in‚0 disk.dm-0.in‚2.46875% memory_percent‚41.224879026412964$ proc.memory.rssMB‚11254.03515625 network.recv‚198020 disk.drbd1584.out‚0 network.sent‚ 2.284571e+06$ disk./.usageGB‚586.3623123168945 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0$ disk.nvme0n1.out‚567.76318359375 disk.drbd2041.out‚0 disk.drbd1550.in‚0 disk.drbd1817.in‚0 disk.drbd1828.in‚0* proc.memory.percent‚17.172294855117798k9Y‰2ó- _wandbruntime‚1074  _runtime‚1074q§¢pÇôÁ % _timestamp‚1.7858113292017217e+09 _runtime‚1074.391665436 _step‚196 & lr-AdamW/pg1‚6.440469402677994e-05 & lr-AdamW/pg2‚6.440469402677994e-05 trainer/global_step‚2499©ƒØWÌõÆ trainer/global_step‚2499 % _timestamp‚1.7858113292017217e+09 _runtime‚1074.391665436 _step‚196 & lr-AdamW/pg1‚6.440469402677994e-05 & lr-AdamW/pg2‚6.440469402677994e-05Ä÷ òV2ö- _wandbruntime‚1074  _runtime‚1074[¬x }÷´O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17011936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2259320953344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16612077 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚42921450200064 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2194020051820 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚43023641238528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚45120940752896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚44049098371776 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚41854975477472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26396749 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 % _timestamp‚1.7858113292026327e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚45120836283904 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚267 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚45120836283904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2199490552832 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41854992516832 _runtime‚1074.396682735 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚42921433160704 WEDeviceStatsMü×öÃ=onitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2199403123200 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26406405 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26396749 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚45282802419712 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚42921433160704 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43018484 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚43023616334848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚42921450200064 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚45120940752896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2199403123200 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16612079 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚44048995529292 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 _step‚197 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16612077 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7166774 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2194105854944 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚43023641238528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚45120940752896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚44049098371776 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚41854975477472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26396749 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 % _timestamp‚1.7858113292026327e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚45120836283904 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚267 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚45120836283904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2199490552832 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41854992516832 _runtime‚1074.396682735 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚42921433160704 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2199403123200 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26406405 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26396749 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚45282802419712 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚42921433160704 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43018484 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚43023616334848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚42921450200064 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚45120940752896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2199403123200 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16612079 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚44048995529292 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 _step‚197 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16612077 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7166774 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2194105854944Å í2ù- _wandbruntime‚1074  _runtime‚1074Y ”Ã&ú½M @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚44066613174834 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2200282742784 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚45138987142656 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚45138882666496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2200370179584 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17018776 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚41871732689152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41871715649792 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9848832 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16618722 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚45301070333952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16618720 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2200370179584 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2200282742784 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚42938599923712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26407272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17018515 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43040823451648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚42938599923712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚44066716017430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2194897525042 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43035664 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7169942 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16618720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43025992 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43035664 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16618722 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚45300908471808 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43040848355328 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚198 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9848834 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43025992 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2194983328278 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42938616963072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1‡¼ÐìûÂM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2194983328278 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42938616963072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚44066613174834 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2200282742784 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚45138987142656 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚45138882666496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2200370179584 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17018776 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚41871732689152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41871715649792 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9848832 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16618722 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚45301070333952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16618720 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2200370179584 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2200282742784 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚42938599923712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26407272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17018515 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43040823451648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚42938599923712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚44066716017430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2194897525042 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43035664 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7169942 M>DeviceStatsMonitor.on_train_batch_end/allocatedáÉÂÜ_bytes.all.peak‚ 2940455936 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16618720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43025992 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43035664 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16618722 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚45300908471808 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43040848355328 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚198 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9848834 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43025992 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712ÆŽ•Ô22ü- _wandbruntime‚1074  _runtime‚1074Gcýü 1loss/SRqqt1/detection_loss‚1.1195965598050854 /loss/FBt1/detection_loss‚10.067331714289528 _step‚199 /loss/FRt1/assignment_loss‚4.369655057003624 1loss/SRqqt2/detection_loss‚3.2926975631378066 /loss/FBt2/detection_loss‚4.0374183264274395 $loss/total_loss‚36.6461639931386 % _timestamp‚1.7858113295566318e+09 0loss/FRt2/assignment_loss‚3.7204625359896957 0loss/FBt2/assignment_loss‚0.1131265118137576 trainer/global_step‚2499 2loss/SRqqt1/assignment_loss‚1.8062707644289102 /loss/FRt1/detection_loss‚3.1517108865082264 .loss/FRt2/detection_loss‚3.109362707740959 epoch‚0 _runtime‚ 1074.74640096 2loss/SRqqt2/assignment_loss‚1.6372172750637566 1loss/FBt1/assignment_loss‚0.22131409092980903gª™þ 0loss/FBt2/assignment_loss‚0.1131265118137576 trainer/global_step‚2499 2loss/SRqqt1/assignment_loss‚1.8062707644289102 /loss/FRt1/detection_loss‚3.1517108865082264 .loss/FRt2/detection_loss‚3.109362707740959 epoch‚0 _runtime‚ 1074.74640096 2loss/SRqqt2/assignment_loss‚1.6372172750637566 1loss/FBt1/assignment_loss‚0.22131409092980903 1loss/SRqqt1/detection_loss‚1.1195965598050854 /loss/FBt1/detection_loss‚10.067331714289528 _step‚199 /loss/FRt1/assignment_loss‚4.369655057003624 1loss/SRqqt2/detection_loss‚3.2926975631378066 /loss/FBt2/detection_loss‚4.0374183264274395 $loss/total_loss‚36.6461639931386 % _timestamp‚1.7858113295566318e+09 0loss/FRt2/assignment_loss‚3.7204625359896957ÇÓ€Ñw‘ÿ‚(:† †£ÅÓغý‚ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚111.792' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.264 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ž«€•_€‚(:Ô  †£ÅÓã¬Àƒ disk.drbd1938.out‚0$ disk./.usageGB‚586.3608894348145 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd1965.in‚0 disk.drbd1817.in‚0 disk.rbd1.in‚0 disk.drbd1356.in‚0 disk.sda.in‚ 0.41015625 disk.sda.out‚ 34.98828125 disk.drbd1965.out‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0' proc.memory.availableMB‚ 38518.71875$ proc.memory.rssMB‚11254.03515625 disk.drbd1828.out‚0 disk.sda1.out‚0 cpu‚13.950460803284658 disk.drbd1931.in‚0 disk.drbd2111.in‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0 network.recv‚200409 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0 disk.drbd2118.in‚0 disk.drbd2041.in‚0 disk.drbd2078.in‚0 disk.drbd1510.out‚0 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 34.98828125 disk.rbd1.out‚ 4.466796875 disk.drbd1226.in‚0 disk.rbd0.out‚0* proc.memory.percent‚17.172294855117798 disk.dm-0.in‚2.46875) disk./.usagePercent‚32.79237050802669 disk.drbd1584.out‚0 disk.drbd1285.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1285.out‚0 disk.sda1.in‚0$ memory_percent‚41.22509956359863 network.sent‚ 2.329544e+06 disk.drbd2041.out‚0 disk.drbd1550.in‚0 disk.drbd1584.in‚0 disk.rbd0.in‚0 disk.dm-0.out‚ 568.16796875! disk.nvme0n1.out‚ 568.16796875 proc.cpu.threads‚29 disk.drbd1226.out‚0 disk.sda2.in‚ 0.41015625 disk.drbd1289.out‚0ªqø‘‚(:† •£ÅÓ¿Ì•ƒ gpu.0.gpu‚34.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚126.597' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚42.199 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ñó-pi‚‚(:Þ  •£ÅÓÉü·ƒ disk.drbd1965.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 4.466796875 disk.nvme0n1.in‚2.46875 disk.drbd1226.in‚0 disk.drbd2118.out‚0 cpu‚13.94646272836748$ disk.nvme0n1.out‚568.42333984375 disk.sda1.in‚0$ disk./.usageGB‚586.3607711791992 disk.drbd1817.out‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd1285.in‚0 disk.drbd1289.in‚0 disk.sda.out‚ 36.15234375 disk.drbd1226.out‚0 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.dm-0.in‚2.46875 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.in‚0 network.recv‚202752 disk.drbd1931.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1550.out‚0 disk.drbd2078.out‚0* proc.memory.availableMB‚38519.21484375 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd1828.in‚0 disk.drbd1510.out‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.sda2.out‚ 36.15234375 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 proc.cpu.threads‚29 network.sent‚ 2.374467e+06 disk.drbd1673.in‚0 disk.drbd1584.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd2041.in‚0 disk.drbd2111.out‚0% memory_percent‚41.224342584609985* disk./.usagePercent‚32.792363894553624 disk.drbd1356.in‚0! disk.dm-0.out‚568.42333984375 disk.drbd1828.out‚0 disk.drbd1510.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1938.in‚0 disk.drbd1931.out‚0 disk.drbd1550.in‚0 disk.drbd1289.out‚0ÁìC-2ƒ- _wandbruntime‚1095  _runtime‚1095ªáƒÇ„Á & lr-AdamW/pg1‚6.569330335104525e-05 & lr-AdamW/pg2‚6.569330335104525e-05 trainer/global_step‚2549 % _timestamp‚1.7858113500405972e+09 _runtime‚1095.230536745 _step‚200¦ÐߧÌ…Æ & lr-AdamW/pg1‚6.569330335104525e-05 & lr-AdamW/pg2‚6.569330335104525e-05 trainer/global_step‚2549 % _timestamp‚1.7858113500405972e+09 _runtime‚1095.230536745 _step‚200È]uèò2†- _wandbruntime‚1095  _runtime‚1095 †S»'‡µO WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26924751 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46023170569728 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17351534 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚42692001132832 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2243486688256 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2243399258624 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚268 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2304375065600 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚42691984093472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10041747 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚44929995451284 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17351266 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2237908515328 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16944227 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _runtime‚ 1095.23462701 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26924751 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26934407 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7309787 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚43883997078528 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚43779771311104 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43878636 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2237994318452 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26934407 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚2549 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚46023275038720 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46023275038720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚43868978 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚44929892608800 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16944229 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚43779788350464 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16944227 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚46188504915456 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚266 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7309521 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2304507836928 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2243486688256 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚43779788350464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚201 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚43883972174848 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46023170569728 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚16944229 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10041745Ìô˜"À'ˆºO TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26934407 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚2549 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚46023275038720 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46023275038720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚43868978 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚44929892608800 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16944229 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚43779788350464 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16944227 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚46188504915456 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚266 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7309521 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2304507836928 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2243486688256 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚43779788350464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚201 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚43883972174848 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46023170569728 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚16944229 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10041745 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26924751 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46023170569728 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17351534 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚42692001132832 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2243486688256 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2243399258624 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚268 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2304375065600 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚42691984093472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10041747 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚44929995451284 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17351266 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2237908515328 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16944227 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _runtime‚ 1095.23462701 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26924751 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26934407 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7309787 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚43883997078528 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚43779771311104 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43878636 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2237994318452 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136ɸÙeI2‰- _wandbruntime‚1095  _runtime‚1095¡hëŠÀM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17358236 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10045684 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚46041321428480 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2244278878208 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚44947510254342 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚44947613096938 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚42708741305152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10045682 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2238785988550 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚43796938074112 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2244366315008 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚43796955113472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚43796938074112 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]KDeviceStatsMonitor.on_train_“³Ûbatch_end/inactive_split_bytes.small_pool.freed‚ 2305280308736 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7312552 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43901179291648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 trainer/global_step‚2549 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚46041216952320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43886144 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26944944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚202 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46041321428480 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26935274 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858113503822124e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16950872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚1095.572473685 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚43796955113472 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2244278878208 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17357991 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚42708724265792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26935274 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43886144 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7312309 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43901204195328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2305408878592 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚46206613073920 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43895816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2238871791786 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43895816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16950870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0p¯9ŒË&‹ÅM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16950870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17358236 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10045684 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚46041321428480 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2244278878208 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚44947510254342 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚44947613096938 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚42708741305152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10045682 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2238785988550 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚43796938074112 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2244366315008 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚43796955113472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚43796938074112 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2305280308736 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7312552 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43901179291648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 trainer/global_step‚2549 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚46041216952320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43886144 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26944944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚202 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46041321428480 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26935274 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858113503822124e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16950872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚1095.572473685 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚43796955113472 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2244278878208 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17357991 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚42708724265792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26935274 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43886144 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7312309 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43901204195328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2305408878592 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚46206613073920 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43895816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2238871791786 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43895816Ê$@Ζ2Œ- _wandbruntime‚1095  _runtime‚1095ìÝp¶ü 1loss/FBt1/assignment_loss‚0.32200918515842536 .loss/FRt1/detection_loss‚3.382364094041396 % _timestamp‚1.7858113503833752e+09 _runtime‚1095.573140264 &loss/total_loss‚36.591537358806605 1loss/FBt2/assignment_loss‚0.09084140753638449 0loss/SRqqt1/detection_loss‚1.251370745684108 1loss/SRqqt2/detection_loss‚2.9248368174214905 _step‚203 1loss/SRqqt1/assignment_loss‚1.512387718577854 .loss/FBt2/detection_loss‚2.928616744154584 trainer/global_step‚2549 epoch‚0 /loss/FRt1/assignment_loss‚4.141023891559546 0loss/FRt2/assignment_loss‚3.7885396745469837 1loss/SRqqt2/assignment_loss‚1.522984292688249 /loss/FRt2/detection_loss‚3.2633556340947565 /loss/FBt1/detection_loss‚11.463207153342823ËÛ ”Ž 1loss/SRqqt1/assignment_loss‚1.512387718577854 .loss/FBt2/detection_loss‚2.928616744154584 trainer/global_step‚2549 epoch‚0 /loss/FRt1/assignment_loss‚4.141023891559546 0loss/FRt2/assignment_loss‚3.7885396745469837 1loss/SRqqt2/assignment_loss‚1.522984292688249 /loss/FRt2/detection_loss‚3.2633556340947565 /loss/FBt1/detection_loss‚11.463207153342823 1loss/FBt1/assignment_loss‚0.32200918515842536 .loss/FRt1/detection_loss‚3.382364094041396 % _timestamp‚1.7858113503833752e+09 _runtime‚1095.573140264 &loss/total_loss‚36.591537358806605 1loss/FBt2/assignment_loss‚0.09084140753638449 0loss/SRqqt1/detection_loss‚1.251370745684108 1loss/SRqqt2/detection_loss‚2.9248368174214905 _step‚203˳{Åš‘‚(:† ¤£ÅÓ½ÿä‚ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚119.817' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚39.939 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0'Ü$h‚(:Ý  ¤£ÅÓÛ©¶ƒ proc.cpu.threads‚29 disk.drbd2111.in‚0 disk.drbd1938.in‚0 disk.rbd1.out‚ 4.4990234375 disk.drbd1965.in‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0$ disk.nvme0n1.out‚569.04248046875 disk.drbd1226.out‚0 disk.drbd1285.out‚0 disk.dm-0.in‚2.46875 disk.nvme0n1.in‚2.46875$ proc.memory.rssMB‚11254.03515625 disk.drbd1584.out‚0 disk.drbd1931.in‚0$ memory_percent‚41.22553467750549 disk.sda2.out‚ 36.6328125 disk.drbd1931.out‚0 disk.drbd1226.in‚0$ disk./.usageGB‚586.3609085083008 disk.drbd2041.in‚0! disk.dm-0.out‚569.04248046875 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.drbd1584.in‚0 disk.rbd0.in‚0* proc.memory.availableMB‚38518.43359375 disk.drbd1289.in‚0 disk.drbd1673.out‚0 disk.sda.in‚ 0.41015625* proc.memory.percent‚17.172294855117798 disk.drbd1510.out‚0 disk.drbd2078.out‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 0.41015625 disk.drbd1828.out‚0 cpu‚13.942672294027536 disk.sda1.in‚0 disk.drbd1550.out‚0 network.sent‚ 2.377537e+06* disk./.usagePercent‚32.792371574715894 disk.drbd1552.out‚0 disk.drbd2041.out‚0 disk.sda.out‚ 36.6328125 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.rbd0.out‚0 network.recv‚203844 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1828.in‚0 disk.drbd1285.in‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd2078.in‚0"ù\½2‘- _wandbruntime‚1116  _runtime‚1116|sggÇ’Á _step‚204 & lr-AdamW/pg1‚6.698191267531055e-05 & lr-AdamW/pg2‚6.698191267531055e-05 trainer/global_step‚2599 % _timestamp‚1.7858113708641226e+09 _runtime‚1116.054068651b›úÌ“Æ & lr-AdamW/pg1‚6.698191267531055e-05 & lr-AdamW/pg2‚6.698191267531055e-05 trainer/global_step‚2599 % _timestamp‚1.7858113708641226e+09 _runtime‚1116.054068651 _step‚204̾`÷b2”- _wandbruntime‚1116  _runtime‚1116†öð¦¼'•¶O SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2349569035264 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚44738788 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚205 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27462409 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _runtime‚1116.055720461 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚46925609324544 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7452629 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17276379 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17691226 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚44638109461504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2287395394048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10238597 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2287482823680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚2599 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚47093897050112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚45810789688308 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚243 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚44744328014848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 % _timestamp‚1.7858113708650765e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27462409 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27452753 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚44638126500864 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚44638126500864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17690983 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚47094050530816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚43528992709472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2281796978836 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2281882781960 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46925504855552 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚44729130 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7452388 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2287395394048 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46925609324544 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46925504855552 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚44638109461504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2287395394048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10238597 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2287482823680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚2599 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚47093897050112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚45810789688308 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚243 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚44744328014848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 % _timestamp‚1.7858113708650765e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27462409 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27452753 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚44638126500864 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚44638126500864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17690983 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚47094050530816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚43528992709472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2281796978836 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2281882781960 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46925504855552 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚44729130 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7452388 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2287395394048 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46925609324544 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46925504855552 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _runtime‚1116.055720461 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032ÍÊn(2—- _wandbruntime‚1116  _runtime‚1116¦tÆ&˜ÀM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2282674452058 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚46943655714304 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2288275013632 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10242534 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚206 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚44761535131648 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2288275013632 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚1116.409401259 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17283020 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚43545749921152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2288362450432 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚45828407333850 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2282760255294 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17283022 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27463276 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17283022 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44755968 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7455456 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2350472254976 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2288362450432 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚44655293263872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27463276 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚43545732881792 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46943655714304 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27472946 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17697754 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10242532 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7455222 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 trainer/global_step‚2599 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚47112158763008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858113712191184e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17697990 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44746296 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚46943551238144 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚45828510176446 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0niË&™ÅM >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27463276 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17283022 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44755968 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7455456 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2350472254976 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2288362450432 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚44655293263872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27463276 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚43545732881792 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46943655714304 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27472946 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17697754 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10242532 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7455222 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 trainer/global_step‚2599 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚47112158763008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858113712191184e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17697990 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44746296 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚46943551238144 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚45828510176446 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2282674452058 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚46943655714304 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2288275013632 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10242534 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚206 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚44761535131648 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2288275013632 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚1116.409401259 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17283020 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚43545749921152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2288362450432 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚45828407333850 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2282760255294 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17283022 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887ÎHã"V2š- _wandbruntime‚1116  _runtime‚1116ÅÆ<ë›û 0loss/SRqqt2/detection_loss‚2.577421382754699 _step‚207 0loss/FRt2/assignment_loss‚3.7810562434403794 2loss/SRqqt1/assignment_loss‚1.4066125381737948 .loss/FBt1/detection_loss‚7.793495774269104 1loss/SRqqt2/assignment_loss‚1.695461094379425 2loss/FBt2/assignment_loss‚0.027122969674564956 /loss/FBt2/detection_loss‚3.0152361724934167 %loss/total_loss‚32.13913607066145 epoch‚0 _runtime‚ 1116.41009628 /loss/FRt1/assignment_loss‚4.056982281116339 0loss/FBt1/assignment_loss‚0.4430978703312576 /loss/FRt1/detection_loss‚3.0540303563078246 .loss/FRt2/detection_loss‚2.879986788591613 1loss/SRqqt1/detection_loss‚1.4086325991290325 trainer/global_step‚2599 % _timestamp‚1.7858113712203307e+09Š)ê^œ€ .loss/FRt2/detection_loss‚2.879986788591613 1loss/SRqqt1/detection_loss‚1.4086325991290325 trainer/global_step‚2599 % _timestamp‚1.7858113712203307e+09 0loss/SRqqt2/detection_loss‚2.577421382754699 _step‚207 0loss/FRt2/assignment_loss‚3.7810562434403794 2loss/SRqqt1/assignment_loss‚1.4066125381737948 .loss/FBt1/detection_loss‚7.793495774269104 1loss/SRqqt2/assignment_loss‚1.695461094379425 2loss/FBt2/assignment_loss‚0.027122969674564956 /loss/FBt2/detection_loss‚3.0152361724934167 %loss/total_loss‚32.13913607066145 epoch‚0 _runtime‚ 1116.41009628 /loss/FRt1/assignment_loss‚4.056982281116339 0loss/FBt1/assignment_loss‚0.4430978703312576 /loss/FRt1/detection_loss‚3.0540303563078246Ï¢±Û>h‚(:Ý  ³£ÅÓˆ²Æƒ disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.rbd1.out‚ 4.5927734375 cpu‚13.939605195056714 disk.drbd1226.in‚0 disk.sda1.out‚0 disk.drbd1817.out‚0 disk.sda2.in‚ 0.41015625 network.sent‚ 2.422481e+06 disk.drbd2041.out‚0 disk.drbd1828.in‚0 disk.drbd1356.out‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0% memory_percent‚41.224974393844604 disk.drbd1510.out‚0 disk.rbd1.in‚0 disk.drbd1226.out‚0 disk.drbd1584.out‚0 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.drbd1289.in‚0 disk.nvme0n1.in‚2.46875* proc.memory.availableMB‚38518.80078125 disk.drbd2078.out‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0! disk.dm-0.out‚569.20654296875 disk.drbd1510.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd2111.in‚0 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.rbd0.in‚0* proc.memory.percent‚17.172294855117798) disk./.usagePercent‚32.79237050802669 disk.drbd1550.in‚0 disk.drbd1285.out‚0 disk.sda1.in‚0 proc.cpu.threads‚29 network.recv‚206213$ disk./.usageGB‚586.3608894348145 disk.sda2.out‚ 37.1484375 disk.drbd1965.in‚0 disk.drbd1584.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1965.out‚0 disk.dm-0.in‚2.46875 disk.drbd1817.in‚0$ disk.nvme0n1.out‚569.20654296875 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.sda.in‚ 0.41015625 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1938.out‚0 disk.sda.out‚ 37.1484375ª{¶3œž‚(:‘ ³£ÅÓáõ¾ƒ gpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.936' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.64533333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0+{_2Ÿ- _wandbruntime‚1136  _runtime‚11360pŽƒÆ À & lr-AdamW/pg1‚6.827052199957586e-05 & lr-AdamW/pg2‚6.827052199957586e-05 trainer/global_step‚2649 $ _timestamp‚1.785811391544664e+09 _runtime‚1136.734624847 _step‚208Ý6­}Ë¡Å _step‚208 & lr-AdamW/pg1‚6.827052199957586e-05 & lr-AdamW/pg2‚6.827052199957586e-05 trainer/global_step‚2649 $ _timestamp‚1.785811391544664e+09 _runtime‚1136.734624847Ðo†S2¢- _wandbruntime‚1136  _runtime‚1136ê+N⣶O MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2331391529472 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27980755 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18030623 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10435447 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚44366001325472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2331478959104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _step‚209 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚45496464651264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18030379 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _runtime‚1136.739630115 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27990411 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2331391529472 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚45496447611904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2394890732032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚47827943610368 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚47827839141376 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10435445 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858113915456276e+09 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚45496447611904 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚47827839141376 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45598940 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27990411 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚45604708758528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚07FÁ—Á'¤»O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10435445 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858113915456276e+09 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚45496447611904 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚47827839141376 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45598940 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27990411 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚45604708758528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2331391529472 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27980755 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18030623 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10435447 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚44366001325472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2331478959104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _step‚209 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚45496464651264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18030379 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _runtime‚1136.739630115 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27990411 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2331391529472 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚45496447611904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2394890732032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚47827943610368 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚47827839141376 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640ÑDEŒ[2¥- _wandbruntime‚1137  _runtime‚1137‡ë¹¯Å&¦¿M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2326562915566 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17615172 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚242 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚45621915875328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚2649 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2332358585856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚210 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚47845990000128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2395803103232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45616120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚44382758537152 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2395661950464 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45606448 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚47845990000128 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45616120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17615172 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚44382741497792 $ _timestamp‚1.785811391898444e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚45621890971648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚47845885523968 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17615170 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚46709407255954 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7597943 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18037083 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10439384 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚210 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚47845990000128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2395803103232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45616120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚44382758537152 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2395661950464 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45606448 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚47845990000128 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45616120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17615172 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚44382741497792 $ _timestamp‚1.78581139189»/¸ºa8444e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚45621890971648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚47845885523968 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17615170 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚46709407255954 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7597943 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18037083 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10439384 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 Ey2²- _wandbruntime‚1157  _runtime‚1157ÕÀf¼'³¶O IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17940679 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17940677 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚45203009941472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚46354785762304 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28518413 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚47572686689808 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚1157.819662803 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚46459092 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7738483 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚48730277896192 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚45203026980832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2440072867840 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚48730277896192 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚48904983985664 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚48905137466368 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18370533 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚46465039694848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚48730173427200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18370780 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10632297 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28518413 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10632295 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚48730173427200 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17940677 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7738238 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚47572583847324 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2369573905852 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2369659708976 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 % _timestamp‚1.7858114126260638e+09 _step‚213 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚46354802801664 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2375475094528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2375387664896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚46354785762304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚46449434 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28508757 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2375475094528 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2439944290816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17940679 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚46354802801664 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024æYp^M%´»O E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚46354802801664 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2375475094528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2375387664896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚46354785762304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚46449434 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28508757 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2375475094528 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2439944290816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17940679 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚46354802801664 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17940679 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17940677 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚45203009941472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚46354785762304 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28518413 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚47572686689808 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚1157.819662803 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚46459092 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7738483 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚48730277896192 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚45203026980832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2440072867840 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚48730277896192 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚48904983985664 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚48905137466368 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18370533 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚46465039694848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚48730173427200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18370780 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10632297 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28518413 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10632295 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚48730173427200 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17940677 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7738238 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚47572583847324 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2369573905852 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2369659708976 F ·‰Dt@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 % _timestamp‚1.7858114126260638e+09 _step‚213 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360ÕÉVt2µ- _wandbruntime‚1158  _runtime‚1158ZÉ8Æ&¶ÀM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2376354721280 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18377220 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚46371952525312 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚46371952525312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46476272 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17947320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28528950 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2376267284480 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚46482246811648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17947322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚214 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10636234 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7740988 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18377507 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚1158.170886294 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28528950 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7741273 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚45219767153152 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚287 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2370451379074 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚48748324285952…§zË&·ÅM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚1158.170886294 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28528950 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7741273 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚45219767153152 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚287 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2370451379074 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚48748324285952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2376354721280 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18377220 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚46371952525312 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚46371952525312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46476272 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17947320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28528950 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2376267284480 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚46482246811648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17947322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚214 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10636234 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7740988 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18377507ÖK¢8ñ2¸- _wandbruntime‚1158  _runtime‚1158š bÁ¹ú .loss/FBt2/detection_loss‚4.310451356939642 /loss/FRt2/assignment_loss‚3.650587867807459 1loss/SRqqt2/assignment_loss‚1.436576643740976 1loss/FBt2/assignment_loss‚0.16252766535131072 .loss/FRt2/detection_loss‚3.106117707343749 epoch‚0 trainer/global_step‚2699 1loss/FBt1/assignment_loss‚0.05936497659567976 _step‚215 0loss/SRqqt1/detection_loss‚1.193752831942459 /loss/FBt1/detection_loss‚12.070962393842638 %loss/total_loss‚37.64990707265645 $ _timestamp‚1.785811412981822e+09 _runtime‚1158.171610896 /loss/FRt1/assignment_loss‚4.095748593280842 2loss/SRqqt1/assignment_loss‚1.7148391857631224 /loss/FRt1/detection_loss‚2.9995215895114007 1loss/SRqqt2/detection_loss‚2.8494562605371723©R‚ºÿ 1loss/SRqqt2/detection_loss‚2.8494562605371723 .loss/FBt2/detection_loss‚4.310451356939642 /loss/FRt2/assignment_loss‚3.650587867807459 1loss/SRqqt2/assignment_loss‚1.436576643740976 1loss/FBt2/assignment_loss‚0.16252766535131072 .loss/FRt2/detection_loss‚3.106117707343749 epoch‚0 trainer/global_step‚2699 1loss/FBt1/assignment_loss‚0.05936497659567976 _step‚215 0loss/SRqqt1/detection_loss‚1.193752831942459 /loss/FBt1/detection_loss‚12.070962393842638 %loss/total_loss‚37.64990707265645 $ _timestamp‚1.785811412981822e+09 _runtime‚1158.171610896 /loss/FRt1/assignment_loss‚4.095748593280842 2loss/SRqqt1/assignment_loss‚1.7148391857631224 /loss/FRt1/detection_loss‚2.9995215895114007×V¼7œ»‚(:‘ à£ÅÓÅÎê‚ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚97.023' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚32.340999999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¡Iß{b¼‚(:×  à£ÅÓ°Ý®ƒ disk.drbd1584.out‚0$ proc.memory.rssMB‚11254.03515625 proc.cpu.threads‚29$ disk./.usageGB‚586.3607063293457 disk.drbd1289.in‚0 disk.nvme0n1.in‚2.46875 disk.sda.out‚ 37.93359375 disk.drbd1965.out‚0 disk.dm-0.out‚570.2529296875 disk.drbd1510.in‚0 disk.drbd1938.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0# disk.nvme0n1.out‚570.2529296875 network.recv‚211988 disk.drbd1285.in‚0 disk.drbd1289.out‚0 disk.rbd1.out‚4.6875 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.drbd1931.out‚0 disk.drbd1226.out‚0) disk./.usagePercent‚32.79236026781033 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.drbd1584.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1931.in‚0 disk.drbd1356.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1550.out‚0 disk.drbd1965.in‚0% memory_percent‚41.226524114608765 disk.rbd0.in‚0 disk.drbd1356.out‚0 cpu‚13.929040572453701 disk.drbd1550.in‚0 disk.drbd1510.out‚0 disk.rbd1.in‚0 disk.drbd2041.in‚0 disk.drbd1226.in‚0* proc.memory.percent‚17.172294855117798 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.drbd2111.in‚0 disk.drbd2041.out‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd1938.out‚0* proc.memory.availableMB‚38517.78515625 network.sent‚ 2.515466e+06 disk.drbd2078.out‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd1828.in‚0 disk.sda2.out‚ 37.93359375(Xaø2½- _wandbruntime‚1178  _runtime‚1178‹2PǾÁ trainer/global_step‚2749 % _timestamp‚1.7858114334628706e+09 _runtime‚1178.652817561 _step‚216 & lr-AdamW/pg1‚7.084774064810645e-05 & lr-AdamW/pg2‚7.084774064810645e-05èp:lÌ¿Æ & lr-AdamW/pg1‚7.084774064810645e-05 & lr-AdamW/pg2‚7.084774064810645e-05 trainer/global_step‚2749 % _timestamp‚1.7858114334628706e+09 _runtime‚1178.652817561 _step‚216Ø[µðû2À- _wandbruntime‚1178  _runtime‚1178ƒ—U;Á¶O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18709774 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2485083414528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29036759 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚46040035596832 _runtime‚1178.657652611 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2419383800320 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47309586 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18272829 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚49810478949376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29036759 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2419383800320 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46040018557472 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2485214088704 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2419471229952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚47325395534848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚48453583769316 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚47213123912704 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2413462369360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚49632612182016 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18272829 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚48453480926832 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚49632507713024 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚49632507713024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚47325420438528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10829147 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚264 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7880629 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚49632612182016 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚49810634527232 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858114334638124e+09 _step‚217 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚47213140952064 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚47213123912704 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632‚#ëÁ'»O =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46040018557472 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2485214088704 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2419471229952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚47325395534848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚48453583769316 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚47213123912704 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2413462369360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚49632612182016 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18272829 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚48453480926832 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚49632507713024 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚49632507713024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚47325420438528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10829147 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚264 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7880629 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚49632612182016 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚49810634527232 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858114334638124e+09 _step‚217 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚47213140952064 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚47213123912704 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18709774 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2485083414528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29036759 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚46040035596832 _runtime‚1178.657652611 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2419383800320 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47309586 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18272829 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚49810478949376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29036759 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2419383800320 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130ÙÿÉ £2Ã- _wandbruntime‚1179  _runtime‚1179ó^ñÆ&ÄÀM ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚46056775769152 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚47336424 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚49650658571776 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18279470 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2485983291904 % _timestamp‚1.7858114338194582e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18716893 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29056952 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚49828585943552 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2486116056064 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47326752 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2420350856704 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2420350856704 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚48471201414970 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚49828743611392 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18716645 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚47230290675712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2420263419904 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29047282 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18279472 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10833082 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚1179.009738829 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚49650658571776 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10833084 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7883563 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18279472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚47342602651648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2414339842582 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47336424 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 trainer/global_step‚2749 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚47230307715072 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚46056758729792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚47230307715072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _step‚218 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29056952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2414425645818 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2420263419904 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29047282 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18279472 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10833082 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚1179.009738829 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚49650658571776 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10833084 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7883563 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18279472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚47342602651648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2414339842582 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47336424 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 trainer/global_step‚2749 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚47230307715072 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚46056758729792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚47230307715072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _step‚218 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29056952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2414425645818 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚49650658571776 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18279470 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2485983291904 % _timestamp‚1.7858114338194582e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18716893 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29056952 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚49828585943552 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2486116056064 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47326752 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2420350856704 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2420350856704 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚48471201414970 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚49828743611392 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18716645 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚47230290675712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816ÚcP¢|2Æ- _wandbruntime‚1179  _runtime‚1179h¸ëИÇý 2loss/FBt2/assignment_loss‚0.014622562987872362 .loss/FBt2/detection_loss‚1.802149753102537 trainer/global_step‚2749 /loss/FRt1/assignment_loss‚4.446010443142482 0loss/FBt1/assignment_loss‚0.6010076608155421 /loss/FRt2/detection_loss‚2.4318174829176806 %loss/total_loss‚35.26542122629882 epoch‚0 _step‚219 /loss/FRt2/assignment_loss‚3.772937473900821 2loss~ºàk/SRqqt1/assignment_loss‚1.4025060489177703 1loss/SRqqt1/detection_loss‚1.0714809859616474 1loss/SRqqt2/detection_loss‚3.0117563819381554 % _timestamp‚1.7858114338206625e+09 2loss/SRqqt2/assignment_loss‚1.4594241781553752 /loss/FRt1/detection_loss‚3.2671916498669558 .loss/FBt1/detection_loss‚11.98451660459198 _runtime‚1179.010477791 ÙOuÈ‚ /loss/FRt2/detection_loss‚2.4318174829176806 %loss/total_loss‚35.26542122629882 epoch‚0 _step‚219 /loss/FRt2/assignment_loss‚3.772937473900821 2loss/SRqqt1/assignment_loss‚1.4025060489177703 1loss/SRqqt1/detection_loss‚1.0714809859616474 1loss/SRqqt2/detection_loss‚3.0117563819381554 % _timestamp‚1.7858114338206625e+09 2loss/SRqqt2/assignment_loss‚1.4594241781553752 /loss/FRt1/detection_loss‚3.2671916498669558 .loss/FBt1/detection_loss‚11.98451660459198 _runtime‚1179.010477791 2loss/FBt2/assignment_loss‚0.014622562987872362 .loss/FBt2/detection_loss‚1.802149753102537 trainer/global_step‚2749 /loss/FRt1/assignment_loss‚4.446010443142482 0loss/FBt1/assignment_loss‚0.6010076608155421Û2õ’¸œÉ‚(:‘ ï£Åәݠƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚116.833' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.94433333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0×°ç[Ê‚(:Ð  ï£ÅÓÔÏȃ disk.drbd1550.out‚0 disk.drbd1817.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1356.out‚0$ disk./.usageGB‚586.3608245849609 disk.sda.out‚38.40625 disk.drbd1938.in‚0 disk.sda1.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1828.out‚0 disk.drbd1673.out‚0 network.recv‚214125) proc.memory.availableMB‚ 38517.9140625 proc.cpu.threads‚29 disk.drbd1226.in‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0$ proc.memory.rssMB‚11254.03515625 network.sent‚ 2.560408e+06( disk./.usagePercent‚32.7923668812834 disk.drbd1289.out‚0 disk.drbd2118.in‚0 disk.drbd1938.out‚0 disk.drbd1931.in‚0 disk.dm-0.in‚2.46875% memory_percent‚41.226327419281006 cpu‚13.925582006193315 disk.rbd0.out‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0# disk.nvme0n1.out‚570.5693359375 disk.drbd1356.in‚0 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.drbd1673.in‚0 disk.drbd1931.out‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd1552.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1226.out‚0 disk.drbd1817.out‚0 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1584.in‚0 disk.drbd1550.in‚0 disk.drbd1510.in‚0 disk.drbd1552.out‚0 disk.sda2.out‚38.40625 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.dm-0.out‚570.5693359375 disk.rbd1.out‚4.78125ß…¢O2Ë- _wandbruntime‚1199  _runtime‚1199Þï©OÇÌÁ _runtime‚1199.526423723 _step‚220 & lr-AdamW/pg1‚7.213634997237176e-05 & lr-AdamW/pg2‚7.213634997237176e-05 trainer/global_step‚2799 % _timestamp‚1.7858114543364894e+09ΪhÂÌÍÆ & lr-AdamW/pg1‚7.213634997237176e-05 & lr-AdamW/pg2‚7.213634997237176e-05 trainer/global_step‚2799 % _timestamp‚1.7858114543364894e+09 _runtime‚1199.526423723 _step‚220ÜC 2Î- _wandbruntime‚1199  _runtime‚1199,ˆ¢¿¼'϶O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18604977 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11025995 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚48071462063104 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚48071479102464 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚50716135578112 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚46877044212832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚50715975805952 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚49334480848824 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _runtime‚1199.530642126 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18604977 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2530224431104 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚48071462063104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11025997 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18604979 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚48071479102464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2463467365376 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29564761 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19050052 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚50534946467840 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29574417 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46877027173472 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚50534841998848 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2457436635992 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2463467365376 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚48185751374848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2463379935744 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚50715975805952 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚49334480848824 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _runtime‚1199.530642126 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18604977 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2530224431104 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚48071462063104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11025997 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18604979 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚48071479102464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2463467365376 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29564761 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19050052 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚50534946467840 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29574417 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46877027173472 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚50534841998848 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2457436635992 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2463467365376 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚48185751374848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2463379935744 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29584954 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29575284 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50734248569856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29575284 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚46893767345792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚2799 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18611620 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚49352098494478 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8026637 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48196576 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2531130313216 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8026885 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚48202958491648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18611620 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19056569 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18611622 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2458228306090 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2464346992128 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚50552888381440 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2531265174528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11029932 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚50552992857600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚49351995651882 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚1199.878358475 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2464346992128 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48186904 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29584954 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚48088628826112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 2ÚHA)48886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48186904 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2464259555328 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚50734088804864 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 $ _timestamp‚1.785811454688073e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19056819 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11029934øí¼žÊ&ÓÄM UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8026885 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚48202958491648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18611620 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19056569 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18611622 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2458228306090 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2464346992128 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚50552888381440 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2531265174528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11029932 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚50552992857600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚49351995651882 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚1199.878358475 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2464346992128 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48186904 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29584954 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚48088628826112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48186904 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2464259555328 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚50734088804864 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 $ _timestamp‚1.785811454688073e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19056819 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11029934 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚48202983395328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29584954 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29575284 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50734248569856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29575284 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚46893767345792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚2799 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18611620 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚49352098494478 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8026637 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48196576 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2531130313216 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2ÞGõl2Ô- _wandbruntime‚1199  _runtime‚1199Z…ûÕú 0loss/FRt1/assignment_loss‚4.4143050956098655 2loss/SRqqt2/assignment_loss‚1.6917501809565645 0loss/FBt2/assignment_loss‚0.2826255853196926 % _timestamp‚1.7858114546893094e+09 2loss/SRqqt1/assignment_loss‚1.6926069045912917 .loss/FRt1/detection_loss‚3.015739302494024 /loss/SRqqt2/detection_loss‚2.83195634735258 /loss/FBt2/detection_loss‚2.4293167690284037 _runtime‚1199.879097486 _step‚223 0loss/FBt1/assignment_loss‚0.2497740641507941 .loss/FBt1/detection_loss‚6.144734099507332 %loss/total_loss‚30.43244117764131 epoch‚0 trainer/global_step‚2799 0loss/FRt2/assignment_loss‚3.6595762157440186 .loss/FRt2/detection_loss‚2.621556146740913 1loss/SRqqt1/detection_loss‚1.3985004661458305­ –6Öÿ 1loss/SRqqt1/detection_loss‚1.3985004661458305 0loss/FRt1/assignment_loss‚4.4143050956098655 2loss/SRqqt2/assignment_loss‚1.6917501809565645 0loss/FBt2/assignment_loss‚0.2826255853196926 % _timestamp‚1.7858114546893094e+09 2loss/SRqqt1/assignment_loss‚1.6926069045912917 .loss/FRt1/detection_loss‚3.015739302494024 /loss/SRqqt2/detection_loss‚2.83195634735258 /loss/FBt2/detection_loss‚2.4293167690284037 _runtime‚1199.879097486 _step‚223 0loss/FBt1/assignment_loss‚0.2497740641507941 .loss/FBt1/detection_loss‚6.144734099507332 %loss/total_loss‚30.43244117764131 epoch‚0 trainer/global_step‚2799 0loss/FRt2/assignment_loss‚3.6595762157440186 .loss/FRt2/detection_loss‚2.621556146740913ßOÛÜxbׂ(:×  þ£ÅÓ½¡Ãƒ disk.drbd1965.out‚0 disk.drbd1552.in‚0 disk.drbd1289.in‚0 disk.sda1.in‚0# disk.nvme0n1.out‚570.9111328125 disk.rbd1.out‚4.84375 disk.nvme0n1.in‚2.46875 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.drbd1552.out‚0 network.sent‚ 2.605451e+06 disk.drbd1285.out‚0 disk.drbd1965.in‚0 proc.cpu.threads‚29$ disk./.usageGB‚586.3608436584473 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.sda2.out‚ 38.76953125 cpu‚13.922409184518873 disk.drbd1931.out‚0 disk.drbd1226.out‚0% memory_percent‚41.226714849472046 disk.sda1.out‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1938.out‚0 disk.drbd2118.in‚0( disk./.usagePercent‚32.7923679479726 disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.sda.out‚ 38.76953125$ proc.memory.rssMB‚11254.03515625 disk.drbd1550.out‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd2078.in‚0 disk.drbd1584.in‚0 disk.rbd0.in‚0 disk.dm-0.out‚570.9111328125 disk.drbd1938.in‚0 disk.drbd1356.out‚0 disk.drbd1931.in‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.out‚0 disk.sda.in‚ 0.41015625 network.recv‚216608 disk.rbd0.out‚0 disk.drbd2041.in‚0* proc.memory.availableMB‚38517.66015625 disk.drbd1817.out‚0 disk.drbd1550.in‚0 disk.drbd2078.out‚0 disk.dm-0.in‚2.46875 disk.drbd2111.in‚0 disk.drbd1356.in‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0 ³ ç›Ø‚(: þ£ÅÓ³²¾ƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚95.459' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚31.81966666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0N8)$›Ù‚(: ¤ÅÓ‹›Øƒ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚98.405' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.80166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0N.mXcÚ‚(:Ø  ¤ÅÓñ´âƒ disk.drbd1226.out‚0 disk.drbd1510.in‚0$ proc.memory.rssMB‚11254.03515625 network.sent‚ 2.650591e+06 disk.drbd1965.in‚0 disk.drbd1931.in‚0# disk./.usageGB‚586.360710144043 disk.drbd2118.in‚0 disk.drbd1550.in‚0* proc.memory.availableMB‚38517.37890625 disk.sda1.out‚0 disk.drbd1673.in‚0 disk.drbd1584.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.dm-0.in‚2.46875 cpu‚13.918815239594771 disk.sda2.in‚ 0.41015625 disk.drbd1550.out‚0 disk.drbd1510.out‚0 disk.sda2.out‚ 39.1015625 disk.sda.out‚ 39.1015625 disk.drbd1285.out‚0 disk.drbd1828.in‚0 proc.cpu.threads‚29 disk.nvme0n1.in‚2.46875! disk.dm-0.out‚571.25244140625 disk.drbd2111.out‚0 disk.rbd1.out‚ 4.9296875 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0$ disk.nvme0n1.out‚571.25244140625$ memory_percent‚41.22714400291443 disk.drbd1938.in‚0 disk.drbd1965.out‚0 disk.drbd1285.in‚0 disk.drbd1289.in‚0 disk.drbd1356.in‚0 disk.drbd1828.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1673.out‚0 disk.drbd1552.in‚0 disk.drbd1356.out‚0 disk.rbd0.out‚0) disk./.usagePercent‚32.79236048114817 disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.drbd2111.in‚0 network.recv‚219007 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd2041.in‚0µM[Ð2Û- _wandbruntime‚1220  _runtime‚1220ýe’ÇÜÁ _step‚224 & lr-AdamW/pg1‚7.342495929663707e-05 & lr-AdamW/pg2‚7.342495929663707e-05 trainer/global_step‚2849 % _timestamp‚1.7858114754367137e+09 _runtime‚1220.628617043W.A±ÌÝÆ % _timestamp‚1.7858114754367137e+09 _runtime‚1220.628617043 _step‚224 & lr-AdamW/pg1‚7.342495929663707e-05 & lr-AdamW/pg2‚7.342495929663707e-05 trainer/global_step‚2849à)Ôð2Þ- _wandbruntime‚1220  _runtime‚1220®À»'ßµO Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚51437176284672 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚51437280753664 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2507376071168 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2507463500800 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2501239296376 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18937127 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚48929800213504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2507376071168 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚51621518375424 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚2849 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49039548 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30102419 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18937129 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚269 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19390796 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11222845 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19390527 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚49046107214848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚51437280753664 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚48929817252864f§‘AÀàºO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8167949 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚50215275085848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2501325099500 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18937127 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30092763 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚48929800213504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2507376071168 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚51621518375424 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚2849 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49039548 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30102419 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18937129 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚269 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19390796 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11222845 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19390527 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚49046107214848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚51437280753664 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚48929817252864 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚51437176284672 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚51437280753664 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2507376071168 QCDeviceStatsMonitoØÜÌr.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2507463500800 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2501239296376 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18937127 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _step‚226 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚48946984015872 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2508255690752 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18943772 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚48946966976512 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2508343127552 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18943772 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49056728 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚48946984015872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚48946966976512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚51455222667264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 trainer/global_step‚2849 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚47730793001152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2508255690752 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚50232995573986 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚50232892731390 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8170746 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚51639786349056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2576447113728 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30103286 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49056728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚51455327143424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18943770 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30103286 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚49047056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19397267 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236ܘYË&ãÅM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _step‚226 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚48946984015872 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2508255690752 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18943772 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚48946966976512 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2508343127552 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18943772 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49056728 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚48946984015872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚48946966976512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚51455222667264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 trainer/global_step‚2849 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚47730793001152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2508255690752 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚50232995573986 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚50232892731390 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8170746 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚51639786349056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2576447113728 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30103286 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49056728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚51455327143424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18943770 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30103286 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚49047056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19397267 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30112956 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2576312252416 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚49063314331648 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0â–ƒg2ä- _wandbruntime‚1220  _runtime‚1220ÿí"åû /loss/FRt2/assignment_loss‚3.780908431695855 2loss/SRqqt1/assignment_loss‚1.7000852662053976 0loss/FBt1/assignment_loss‚0.7157487178822824 &loss/total_loss‚28.688832851797436 % _timestamp‚1.7858114757908914e+09 /loss/FRt1/assignment_loss‚4.106303562091876 /loss/FRt1/detection_loss‚2.8769228308449817 .loss/FBt1/detection_loss‚5.290865168968836 _step‚227 1loss/FBt2/assignment_loss‚0.03215051301333752 -loss/FRt2/detection_loss‚2.78780653036159 1loss/SRqqt1/detection_loss‚1.2878568162319595 0loss/SRqqt2/detection_loss‚2.732851592078805 /loss/FBt2/detection_loss‚1.7927165991143466 _runtime‚1220.980980328 2loss/SRqqt2/assignment_loss‚1.5846168233081699 epoch‚0 trainer/global_step‚2849‹.Æaæ€ /loss/FRt1/assignment_loss‚4.106303562091876 /loss/FRt1/detection_loss‚2.8769228308449817 .loss/FBt1/detection_loss‚5.290865168968836 _step‚227 1loss/FBt2/assignment_loss‚0.03215051301333752 -loss/FRt2/detection_loss‚2.78780653036159 1loss/SRqqt1/detection_loss‚1.2878568162319595 0loss/SRqqt2/detection_loss‚2.732851592078805 /loss/FBt2/detection_loss‚1.7927165991143466 _runtime‚1220.980980328 2loss/SRqqt2/assignment_loss‚1.5846168233081699 epoch‚0 trainer/global_step‚2849 /loss/FRt2/assignment_loss‚3.780908431695855 2loss/SRqqt1/assignment_loss‚1.7000852662053976 0loss/FBt1/assignment_loss‚0.7157487178822824 &loss/total_loss‚28.688832851797436 % _timestamp‚1.7858114757908914e+09ã}õœç‚(:‘ œ¤ÅÓ¼¿Žƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚102.898' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚34.29933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ßA×—_è‚(:Ô  œ¤ÅÓä¿Ðƒ disk.drbd1828.in‚0) proc.memory.availableMB‚ 38517.5703125 disk.drbd2041.in‚0$ memory_percent‚41.22685194015503 network.sent‚ 2.653668e+06 disk.sda1.in‚0 disk.sda2.out‚ 39.55078125 proc.cpu.threads‚29 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1965.out‚0 disk.rbd0.in‚0 disk.drbd1285.in‚0) disk./.usagePercent‚32.79238053490521$ disk./.usageGB‚586.3610687255859 disk.sda.out‚ 39.55078125 disk.sda1.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1931.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd1552.out‚0 disk.drbd1226.in‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0 disk.rbd0.out‚0 disk.rbd1.in‚0! disk.nvme0n1.out‚ 572.08203125* proc.memory.percent‚17.172294855117798 disk.dm-0.out‚ 572.08203125 disk.drbd1289.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 network.recv‚219923 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.dm-0.in‚2.46875 disk.drbd1510.in‚0 disk.drbd2078.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd1550.out‚0 disk.drbd1938.out‚0 disk.drbd2041.out‚0$ proc.memory.rssMB‚11254.03515625 cpu‚13.915603387476851 disk.drbd1828.out‚0 disk.sda2.in‚ 0.41015625 disk.rbd1.out‚ 4.9921875 disk.drbd1584.in‚0 disk.drbd2111.out‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0„Þ,Õ2é- _wandbruntime‚1241  _runtime‚1241ìuïÇêÁ _runtime‚1241.615534252 _step‚228 & lr-AdamW/pg1‚7.471356862090237e-05 & lr-AdamW/pg2‚7.471356862090237e-05 trainer/global_step‚2899 % _timestamp‚1.7858114964255862e+09L.ƒ†ÌëÆ & lr-AdamW/pg1‚7.471356862090237e-05 & lr-AdamW/pg2‚7.471356862090237e-05 trainer/global_step‚2899 % _timestamp‚1.7858114964255862e+09 _runtime‚1241.615534252 _step‚228äG‡ 2ì- _wandbruntime‚1241  _runtime‚1241"z8]° íµO F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚52339615039488 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 124382720 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚229 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2551372206592 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2620595485696 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2545213563008 trainer/global_step‚2899 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19269277 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚49788155403264 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚49906463054848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚48551061444832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚49788138363904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2551372206592 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚52339615039488 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚52527058540544 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30630421 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30620765 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8311905 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 _runtime‚1241.616747373 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.l79% arge_pool.peak‚1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19731372 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11419697 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚52339510570496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19269279 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30620765 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2551459636224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 149286400 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30630421 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚49906487958528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚230 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚49788138363904 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49899700 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚51096275007840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2551459636224 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚228 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19269279 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19731602 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2620719868416 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚48551044405472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚49788155403264 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚52527207826944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚52339510570496 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19269277 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49890042 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11419695 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2545127759884P²ŽÀ'îºO F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30620765 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8311905 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 _runtime‚1241.616747373 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19731372 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11419697 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚52339510570496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19269279 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30620765 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2551459636224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 149286400 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30630421 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚49906487958528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚230 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚49788138363904 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49899700 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚51096275007840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2551459636224 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚228 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19269279 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19731602 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2620719868416 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚48551044405472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚49788155403264 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚52527207826944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚52339510570496 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19269277 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49890042 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11419695 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2545127759884 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚52339615039488 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 124382720 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚229 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2551372206592 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2620595485696 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2545213563008 trainer/global_step‚2899 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19269277 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚49788155403264 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚49906463054848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚48551061444832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚49788138363904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2551372206592 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚52339615039488 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚52527058540544 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30630421 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656ål0@2ï- _wandbruntime‚1241  _runtime‚1241òKÞ¹Å&ð¿M E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚52357661429248 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚49923695075328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11423632 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8314536 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49916880 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30640958 trainer/global_step‚2899 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19275922 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚49907208 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚49805305126912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30631288 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚51113789810898 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2546005233106 $ _timestamp‚1.785811496778112e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚249 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30640958 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚49805322166272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚48567801617152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19275920 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚49805322166272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19275920 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30631288 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚49805305126912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚51113892653494 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49907208 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19275922 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚52357661429248 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2621502629376 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚52357556953088 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2552251826176 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚48567784577792 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚52545172801024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2621635393536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚52545330468864 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49916880 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19738168 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚52357556953088 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2552251826176 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚48567784577792 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚52545172801024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2621635393536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚52545330468864 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49916880 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19738168 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49916880 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30640958 trainer/global_step‚2899 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19275922 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚49907208 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚49805305126912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30631288 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚51113789810898 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2546005233106 $ _timestamp‚1.785811496778112e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚ûSáݯ249 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30640958 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚49805322166272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚48567801617152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19275920 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚49805322166272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19275920 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30631288 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚49805305126912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚51113892653494 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49907208 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19275922 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚52357661429248æ“í¦2ò- _wandbruntime‚1241  _runtime‚1241‡¿«Sóþ 1loss/SRqqt2/detection_loss‚3.2167744458683076 _runtime‚1241.969109347 _step‚231 /loss/FRt2/assignment_loss‚3.747581744735891 2loss/SRqqt2/assignment_loss‚1.7407418152978342 0loss/FBt1/assignment_loss‚0.6547933813805381 .loss/FBt1/detection_loss‚7.074873944123586 /loss/FBt2/detection_loss‚3.2608126098656847 trainer/global_step‚2899 2loss/SRqqt1/assignment_loss‚1.6503129280022075 &loss/total_loss‚32.821049798133316 epoch‚0 % _timestamp‚1.7858114967792873e+09 2loss/FBt2/assignment_loss‚0.043864045588634326 1loss/SRqqt1/detection_loss‚1.1521234521648014 /loss/FRt1/assignment_loss‚4.325088906979215 .loss/FRt1/detection_loss‚3.169923700284267 /loss/FRt2/detection_loss‚2.7841588238423522q·s¥ ôƒ trainer/global_step‚2899 2loss/SRqqt1/assignment_loss‚1.6503129280022075 &loss/total_loss‚32.821049798133316 epoch‚0 % _timestamp‚1.7858114967792873e+09 2loss/FBt2/assignment_loss‚0.043864045588634326 1loss/SRqqt1/detection_loss‚1.1521234521648014 /loss/FRt1/assignment_loss‚4.325088906979215 .loss/FRt1/detection_loss‚3.169923700284267 /loss/FRt2/detection_loss‚2.7841588238423522 1loss/SRqqt2/detection_loss‚3.2167744458683076 _runtime‚1241.969109347 _step‚231 /loss/FRt2/assignment_loss‚3.747581744735891 2loss/SRqqt2/assignment_loss‚1.7407418152978342 0loss/FBt1/assignment_loss‚0.6547933813805381 .loss/FBt1/detection_loss‚7.074873944123586 /loss/FBt2/detection_loss‚3.2608126098656847çs’vœõ‚(:‘ «¤ÅÓ‰©ƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚112.645' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.54833333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0õ»šêbö‚(:×  «¤ÅÓơŃ disk.drbd1965.out‚0) proc.memory.availableMB‚ 38517.7421875* proc.memory.percent‚17.172294855117798 disk.drbd1289.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.in‚0$ disk./.usageGB‚586.3488922119141 disk.drbd1673.in‚0 disk.drbd2041.in‚0$ proc.memory.rssMB‚11254.03515625 proc.cpu.threads‚29 disk.drbd1965.in‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.rbd0.in‚0 disk.drbd1510.in‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.sda2.out‚ 41.0859375) disk./.usagePercent‚32.79169956051722 disk.drbd1938.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 network.sent‚ 2.69861e+06$ disk.nvme0n1.out‚577.87158203125 disk.drbd1356.in‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd1550.out‚0 disk.drbd1828.in‚0 disk.drbd2111.out‚0 cpu‚13.912368475575002 disk.sda.out‚ 41.0859375 disk.drbd1226.out‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.rbd1.out‚ 5.0859375 disk.rbd0.out‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd1226.in‚0 network.recv‚222394 disk.sda1.in‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.41015625 disk.drbd2078.out‚0 disk.drbd1584.out‚0 disk.drbd1817.in‚0 disk.drbd2041.out‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0$ memory_percent‚41.22658967971802 disk.drbd1552.out‚0 disk.drbd1550.in‚0! disk.dm-0.out‚577.87158203125 disk.drbd1817.out‚0 disk.drbd1931.out‚0°e]ä‘÷‚(:† º¤ÅÓ“åä‚ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚121.035' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚40.345 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0.Rvdø‚(:Ù  º¤ÅÓÖ¼©ƒ network.recv‚224660 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.drbd2041.out‚0$ memory_percent‚41.22697114944458 disk.sda2.out‚ 42.81640625 disk.rbd1.out‚ 5.2421875 disk.drbd2078.out‚0) proc.memory.availableMB‚ 38517.4921875* disk./.usagePercent‚32.791732841220394 disk.dm-0.out‚581.3134765625 disk.sda1.in‚0 disk.drbd1584.in‚0 disk.rbd0.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.41015625 disk.sda.out‚ 42.81640625 disk.drbd1285.out‚0$ disk./.usageGB‚586.3494873046875 cpu‚13.909115249093404 disk.drbd1673.out‚0 disk.drbd1550.out‚0 disk.drbd1552.in‚0 disk.drbd1965.in‚0 network.sent‚ 2.743566e+06 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1226.out‚0 disk.drbd1285.in‚0* proc.memory.percent‚17.172294855117798 disk.nvme0n1.in‚2.46875 disk.rbd0.in‚0 disk.drbd1817.out‚0 disk.drbd1965.out‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.41015625 disk.rbd1.in‚0 disk.drbd1584.out‚0 disk.drbd1828.out‚0 proc.cpu.threads‚29 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 disk.drbd2041.in‚0 disk.drbd1226.in‚0 disk.dm-0.in‚2.46875 disk.drbd1289.out‚0 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd1828.in‚0 disk.drbd1552.out‚0 disk.drbd1938.out‚0# disk.nvme0n1.out‚581.3134765625v}Á­2ù- _wandbruntime‚1262  _runtime‚1262C=|ÇúÁ _runtime‚1262.470653124 _step‚232 & lr-AdamW/pg1‚7.600217794516768e-05 & lr-AdamW/pg2‚7.600217794516768e-05 trainer/global_step‚2949 % _timestamp‚1.7858115172792838e+09Þoš…ÌûÆ _step‚232 & lr-AdamW/pg1‚7.600217794516768e-05 & lr-AdamW/pg2‚7.600217794516768e-05 trainer/global_step‚2949 % _timestamp‚1.7858115172792838e+09 _runtime‚1262.470653124èêäjr2ü- _wandbruntime‚1262  _runtime‚1262§õ‹ ¼'ý¶O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚53241844856320 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19601429 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11616545 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2665939594752 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8455206 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31148767 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8454954 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚51977172087348 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2589016223392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 trainer/global_step‚2949 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚53241844856320 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2595368342016 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚50646476514304 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2595455771648 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚50646493553664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 _step‚233 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31158423 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49388053021472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858115172816508e+09 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2595368342016 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚53241949325312 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31148767 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1262.471820562 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19601429 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2595455771648 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚53241949325312 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚50646493553664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2589102026516 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19601427 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20071499 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚53241844856320 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2595368342016 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚50646476514304 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2595455771648 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚50646493553664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 _step‚233 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31158423 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49388053021472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858115172816508e+09 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2595368342016 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚53241949325312 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31148767 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1262.471820562 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19601429 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2595455771648 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚53241949325312 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚50646493553664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2589102026516 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19601427 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20071499 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 I+¿ƒ € ÀM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 166056448 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚51994686890406 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚50663643277312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50777032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11620484 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚50784026011648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚50663660316672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19608070 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19608072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚50784050915328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2666697304576 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚53259995715072 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2596247961600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50767360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚50663643277312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚53259995715072 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2666838457344 % _timestamp‚1.7858115176492934e+09 _runtime‚1262.839573852 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50767360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19608072 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2596335398400 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 141152768 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31159290 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20078272 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8457790 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2589893696614 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19608070 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31168960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392>ÃË& ÅM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚50663643277312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚53259995715072 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2666838457344 % _timestamp‚1.7858115176492934e+09 _runtime‚1262.839573852 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50767360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19608072 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2596335398400 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 141152768 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31159290 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20078272 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8457790 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2589893696614 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19608070 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31168960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 166056448 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚51994686890406 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚50663643277312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50777032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11620484 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚50784026011648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚50663660316672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19608070 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19608072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚50784050915328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2666697304576 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚53259995715072 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2596247961600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50767360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚31686425 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _runtime‚1283.193414437 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚52857966324372 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚51610346 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19933579 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2639451907072 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 % _timestamp‚1.7858115380032635e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚54144283611136 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚51504831704064 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚50225078676832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2632990490024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚51504814664704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19933579 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚50225061637472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8597709 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚54338301945344 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54144283611136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚51627199638528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8597952 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚52858069166856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚51627174734848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2632904686900 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51620004 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54144179142144 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2639364477440 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚51504831704064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2639364477440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2711102306816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392"¶nE2Œ »O TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚51504831704064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2639364477440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2711102306816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceS¤Ïºj%tatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚31686425 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _runtime‚1283.193414437 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚52857966324372 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚51610346 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19933579 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2639451907072 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 % _timestamp‚1.7858115380032635e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚54144283611136 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚51504831704064 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚50225078676832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2632990490024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚51504814664704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19933579 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚50225061637472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8597709 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚54338301945344 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54144283611136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚51627199638528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8597952 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚52858069166856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚51627174734848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2632904686900 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51620004 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54144179142144 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2639364477440í:râ2 - _wandbruntime‚1283  _runtime‚1283.AÆ&Ž ÀM \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2633867963358 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚54162330000896 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 141152768 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚238 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚50241818849152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2640244097024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19940220 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2640331533824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 % _timestamp‚1.7858115383555348e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚51521981427712 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8600563 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19940222 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8600814 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2640331533824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚51644381851648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2711872468480 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚51521998467072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _runtime‚1283.545809814 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51637184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚51644406755328 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51637184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51627512 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31696962 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31687292 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20417895 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20418148 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚51521998467072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚54162330000896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚54356420376576 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51627512 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2640331533824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 % _timestamp‚1.7858115383555348e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚51521981427712 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8600563 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19940222 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8600814 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2640331533824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚51644381851648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2711872468480 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚51521998467072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _runtime‚1283.545809814 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51637184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚51644406755328 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51637184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51627512 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31696962 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31687292 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20417895 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20418148 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚51521998467072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚54162330000896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚54356420376576 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51627512 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚50241818849152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251îÅ<2 - _wandbruntime‚1283  _runtime‚1283 Þ‘ ú /loss/FRt2/assignment_loss‚3.795518227112599 1loss/FBt2/assignment_loss‚0.13113512581549003 /loss/FBt2/detection_loss‚2.8233211217688337 epoch‚0 _runtime‚1283.546540085 /loss/FRt1/assignment_loss‚4.149394291970465 .loss/FRt2/detection_loss‚2.977880474848625 -loss/FBt1/detection_loss‚8.23855745792389 trainer/global_step‚2999 % _timestamp‚1.7858115383567343e+09 2loss/SRqqt1/assignment_loss‚1.5681573386545535 $loss/total_loss‚32.6911918994897 _step‚239 2loss/SRqqt2/assignment_loss‚1.5618780641884045 0loss/FBt1/assignment_loss‚0.6665486026451819 /loss/FRt1/detection_loss‚2.9967861773653164 1loss/SRqqt1/detection_loss‚0.9166277793900924 1loss/SRqqt2/detection_loss‚2.8653872378062513üóeÑ’ ÿ .loss/FRt2/detection_loss‚2.977880474848625 -loss/FBt1/detection_loss‚8.23855745792389 trainer/global_step‚2999 % _timestamp‚1.7858115383567343e+09 2loss/SRqqt1/assignment_loss‚1.5681573386545535 $loss/total_loss‚32.6911918994897 _step‚239 2loss/SRqqt2/assignment_loss‚1.5618780641884045 0loss/FBt1/assignment_loss‚0.6665486026451819 /loss/FRt1/detection_loss‚2.9967861773653164 1loss/SRqqt1/detection_loss‚0.9166277793900924 1loss/SRqqt2/detection_loss‚2.8653872378062513 /loss/FRt2/assignment_loss‚3.795518227112599 1loss/FBt2/assignment_loss‚0.13113512581549003 /loss/FBt2/detection_loss‚2.8233211217688337 epoch‚0 _runtime‚1283.546540085 /loss/FRt1/assignment_loss‚4.149394291970465ï‡ûo ?“ ‚(:Ô  ؤÅәƅ… disk.drbd1510.in‚0 disk.drbd1510.out‚0 network.sent‚ 2.791598e+06 network.recv‚228075$ disk./.usageGB‚586.3494529724121 disk.drbd1938.in‚0# disk.nvme0n1.out‚581.9873046875 disk.rbd0.out‚0 disk.drbd1552.in‚0) proc.memory.availableMB‚ 38517.8046875 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.drbd1584.in‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1550.out‚0 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.dm-0.out‚581.9873046875 disk.drbd2118.out‚0) disk./.usagePercent‚32.79173092117983 disk.nvme0n1.in‚2.46875 disk.drbd1584.out‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 disk.drbd1817.out‚0 disk.sda2.out‚ 43.734375 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd1965.in‚0 disk.sda1.in‚0 proc.cpu.threads‚29 disk.drbd2041.in‚0 disk.sda.out‚ 43.734375 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1552.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1226.out‚0 disk.dm-0.in‚2.46875 disk.rbd1.in‚0 disk.drbd1931.out‚0 cpu‚13.904256953119189 disk.drbd1931.in‚0 disk.drbd2041.out‚0$ memory_percent‚41.22649431228638* proc.memory.percent‚17.172294855117798 disk.drbd1673.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1938.out‚0 disk.rbd1.out‚ 52Ëé1 .3046875 disk.drbd2078.in‚0%ah›” ‚(: ؤÅÓ²½±… gpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚96.469' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.15633333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0àË_2• - _wandbruntime‚1304  _runtime‚1304¿ViÇ– Á & lr-AdamW/pg1‚7.857939659369829e-05 & lr-AdamW/pg2‚7.857939659369829e-05 trainer/global_step‚3049 % _timestamp‚1.7858115592193444e+09 _runtime‚1304.409282928 _step‚2408Øà]Ì— Æ trainer/global_step‚3049 % _timestamp‚1.7858115592193444e+09 _runtime‚1304.409282928 _step‚240 & lr-AdamW/pg1‚7.857939659369829e-05 & lr-AdamW/pg2‚7.857939659369829e-05ðbï‡!2˜ - _wandbruntime‚1304  _runtime‚1304´+ÇG¼'™ ¶O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32204771 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚52363152815104 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2683448042496 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚52363169854464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚253 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚53738863403880 trainer/global_step‚3049 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32214427 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20265729 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20265727 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚55046617896960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2683448042496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2683360612864 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚51062087292832 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚52363152815104 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2683360612864 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2756230586880 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12010247 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8740082 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚55243622098944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2676878953532 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚55046513427968 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52480156 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12010245 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8740333 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32214427 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚51062070253472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2676793150408 % _timestamp‚1.7858115592202635e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚55243786065408 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚52363169854464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚52487555478528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚55046617896960 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12010245 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8740333 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32214427 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚51062070253472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2676793150408 % _timestamp‚1.7858115592202635e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚55243786065408 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚52363169854464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚52487555478528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚55046617896960 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚55046513427968 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52480156 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _runtime‚1304.763187247 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚3049 _step‚242 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2677756426866 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12014184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32224964 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2684240232448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚55064664286720 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20272370 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8742890 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚52504762595328 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚52504737691648 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚52380319578112 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚52380336617472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚53756481049422 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2677670623630 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20757072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53756583892018 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52497336 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20272370 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8743150 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚51078827465152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52497336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2684327669248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52487664 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚55261897009152 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2756995358208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20272372 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2757134413824Ã÷m_Œ ÅM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8742890 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚52504762595328 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚52504737691648 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚52380319578112 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚ÄÓ>Ù?!251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚52380336617472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚53756481049422 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2677670623630 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20757072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53756583892018 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52497336 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20272370 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8743150 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚51078827465152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52497336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2684327669248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52487664 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚55261897009152 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2756995358208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20272372 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2757134413824 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52487664 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _runtime‚1304.763187247 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚3049 _step‚242 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2677756426866 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12014184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32224964 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2684240232448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚55064664286720 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20272370òŠµ´2ž - _wandbruntime‚1304  _runtime‚1304™ãÿŸ ù epoch‚0 1loss/FBt1/assignment_loss‚0.44629880798877114 2loss/FBt2/assignment_loss‚0.021226209513612128 /loss/FRt2/detection_loss‚2.6435133164287894 1loss/SRqqt1/detection_loss‚1.5219579837858688 .loss/FBt1/detection_loss‚7.930928097830878 % _timestamp‚1.7858115595741224e+09 _runtime‚1304.763915698 .loss/FRt1/assignment_loss‚4.15640978497195 1loss/SRqqt1/assignment_loss‚1.617616702403341 1loss/SRqqt2/assignment_loss‚1.570310676243247 .loss/FRt1/detection_loss‚2.550825763179595 0loss/SRqqt2/detection_loss‚2.361751592377337 trainer/global_step‚3049 _step‚243 /loss/FRt2/assignment_loss‚3.786797279952675 .loss/FBt2/detection_loss‚4.293912360945261 %loss/total_loss‚32.90154857562133‚˜Â  þ _step‚243 /loss/FRt2/assignment_loss‚3.786797279952675 .loss/FBt2/detection_loss‚4.293912360945261 %loss/total_loss‚32.90154857562133 epoch‚0 1loss/FBt1/assignment_loss‚0.44629880798877114 2loss/FBt2/assignment_loss‚0.021226209513612128 /loss/FRt2/detection_loss‚2.6435133164287894 1loss/SRqqt1/detection_loss‚1.5219579837858688 .loss/FBt1/detection_loss‚7.930928097830878 % _timestamp‚1.7858115595741224e+09 _runtime‚1304.763915698 .loss/FRt1/assignment_loss‚4.15640978497195 1loss/SRqqt1/assignment_loss‚1.617616702403341 1loss/SRqqt2/assignment_loss‚1.570310676243247 .loss/FRt1/detection_loss‚2.550825763179595 0loss/SRqqt2/detection_loss‚2.361751592377337 trainer/global_step‚3049óÊØM¡ ‚(:’ ç¤ÅÓÛ௃ gpu.0.gpu‚24.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚100.741' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚33.580333333333336 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0<ÕÜ f¢ ‚(:Û  ç¤ÅÓáÕƒ disk.sda.out‚ 44.11328125 disk.drbd1938.out‚0 disk.drbd1584.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0* proc.memory.availableMB‚38516.70703125 disk.sda.in‚ 0.41015625 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.rbd1.out‚ 5.3984375 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.sda1.in‚0 disk.sda2.out‚ 44.11328125 disk.nvme0n1.in‚2.46875 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.drbd1289.in‚0 disk.drbd1356.in‚0 disk.drbd1550.out‚0* disk./.usagePercent‚32.791785962342765 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1931.in‚0# disk.nvme0n1.out‚582.1826171875 network.sent‚ 2.836534e+06 disk.drbd2111.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd1965.out‚0 proc.cpu.threads‚29 disk.drbd1828.in‚0 disk.drbd2041.in‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 cpu‚13.901224321626756 disk.dm-0.out‚582.1826171875 disk.drbd1510.in‚0 disk.drbd1817.out‚0 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.drbd1356.out‚0 disk.drbd1510.out‚0 network.recv‚230376$ disk./.usageGB‚586.3504371643066 disk.drbd2041.out‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0$ proc.memory.rssMB‚11254.03515625* proc.memory.percent‚17.172294855117798 disk.drbd1550.in‚0% memory_percent‚41.228169202804565 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 disk.sda1.out‚0 disk.drbd1552.out‚0ˆ0zM£ ‚(:… ö¤ÅÓú㣃 gpu.0.gpu‚31.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚99.843' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚33.281 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 Ðå*b¤ ‚(:×  ö¤ÅÓ†ì׃ disk.drbd1938.in‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd1673.in‚0 disk.sda2.in‚ 0.41015625 disk.rbd0.out‚0 disk.drbd2041.out‚0 disk.drbd2118.in‚0 disk.drbd1938.out‚0 disk.drbd1584.in‚0 disk.rbd0.in‚0 disk.drbd1552.out‚0 network.recv‚232800 disk.rbd1.in‚0$ disk.nvme0n1.out‚582.61767578125 disk.drbd1356.out‚0 disk.drbd1550.in‚0! disk.dm-0.out‚582.61767578125 disk.drbd2111.out‚0$ proc.memory.rssMB‚11254.03515625 disk.sda2.out‚ 44.37890625 disk.drbd2118.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1356.in‚0( proc.memory.availableMB‚ 38516.921875 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 cpu‚13.898951226989048) disk./.usagePercent‚32.79178468231572* proc.memory.percent‚17.172294855117798# memory_percent‚41.2278413772583 disk.drbd1931.out‚0 disk.sda.in‚ 0.41015625# disk./.usageGB‚586.350414276123 disk.drbd1584.out‚0 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1931.in‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd1285.out‚0 disk.drbd1226.in‚0 disk.drbd1510.in‚0 disk.sda1.in‚0 proc.cpu.threads‚29 network.sent‚ 2.881817e+06 disk.rbd1.out‚ 5.4609375 disk.sda.out‚ 44.37890625 disk.drbd1828.in‚0 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.drbd1289.in‚0 disk.drbd2041.in‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0Ùˆû2¥ - _wandbruntime‚1324  _runtime‚1324 ˜zĦ ¾ _step‚244 % lr-AdamW/pg1‚7.98680059179636e-05 % lr-AdamW/pg2‚7.98680059179636e-05 trainer/global_step‚3099 % _timestamp‚1.7858115796751714e+09 _runtime‚ 1324.86510481$}}ɧ à % lr-AdamW/pg1‚7.98680059179636e-05 % lr-AdamW/pg2‚7.98680059179636e-05 trainer/global_step‚3099 % _timestamp‚1.7858115796751714e+09 _runtime‚ 1324.86510481 _step‚244ô[îÄ…2¨ - _wandbruntime‚1324  _runtime‚1324^’óé¼'© ¶O BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32732773 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12207095 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32742429 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20597877 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚56149326756352 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2727356748288 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32742429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21089616 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚53221508004864 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚55948847713792 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32732773 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2727444177920 _step‚245 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚51899095908832 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 124382720 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2720681613916 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2727356748288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚53347886414848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚55948847713792 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚53221508004864 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144ªmà‹ˆª »O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21089616 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚53221508004864 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚55948847713792 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32732773 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2727444177920 _step‚245 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚51899095908832 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 124382720 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2720681613916 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2727356748288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚53347886414848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚55948847713792 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚53221508004864 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32732773 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12207095 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32742429 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20597877 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚56149326756352 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2727356748288 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32742429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0õÿ’=Ý2« - _wandbruntime‚1325  _runtime‚1325“P*—Æ&¬ ÀM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚3099 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2728323804672 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚53365093531648 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚51915836081152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21096050 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2728323804672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2721559087138 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12211034 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚54637480971526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2728236367872 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53357488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚53238657728512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2728236367872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2802192726528 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚51915819041792 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32752966 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚56167443926016 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _step‚246 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚53238657728512 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53347816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚1325.212670024 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚55966894096384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚56167286258176 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚53365118435328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8885254 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚55966998572544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20604520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚53238674767872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚238 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8885018 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚54637378128930 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20604522 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53347816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20604520 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21096288 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251•rqäË&­ ÅM E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2721559087138 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12211034 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚54637480971526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2728236367872 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53357488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚53238657728512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2728236367872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2802192726528 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚51915819041792 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32752966 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚56167443926016 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _step‚246 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚53238657728512 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53347816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚1325.212670024 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚55966894096384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚56167286258176 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚53365118435328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8885254 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚55966998572544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20604520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚53238674767872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚238 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8885018 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚54637378128930 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20604522 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53347816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20604520 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21096288 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚3099 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2728323804672 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚53365093531648 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚51915836081152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21096050 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2728323804672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680öc –2® - _wandbruntime‚1325  _runtime‚1325°2kC¯ ÿ epoch‚0 1loss/FBt2/assignment_loss‚0.06286837618389048 1loss/SRqqt1/detection_loss‚0.8822486302989645 1loss/SRqqt2/detection_loss‚2.9179374256304333 .loss/FBt1/detection_loss‚5.124864406055874 trainer/global_step‚3099 _runtime‚1325.213413475 0loss/FRt2/assignment_loss‚3.5694958615577086 2loss/SRqqt1/assignment_loss‚1.5023600637591144 _step‚247 &loss/total_loss‚27.031347089642612 0loss/FRt1/assignment_loss‚3.6296704313648283 2loss/SRqqt2/assignment_loss‚1.5198243135024632 /loss/FRt2/detection_loss‚2.7302008517857255 .loss/FBt2/detection_loss‚1.899126038047205 1loss/FBt1/assignment_loss‚0.20902151000660119 .loss/FRt1/detection_loss‚2.983729181449805 % _timestamp‚1.7858115800236232e+09íoÉ1 ° „ &loss/total_loss‚27.031347089642612 0loss/FRt1/assignment_loss‚3.6296704313648283 2loss/SRqqt2/assignment_loss‚1.5198243135024632 /loss/FRt2/detection_loss‚2.7302008517857255 .loss/FBt2/detection_loss‚1.899126038047205 1loss/FBt1/assignment_loss‚0.20902151000660119 .loss/FRt1/detection_loss‚2.983729181449805 % _timestamp‚1.7858115800236232e+09 epoch‚0 1loss/FBt2/assignment_loss‚0.06286837618389048 1loss/SRqqt1/detection_loss‚0.8822486302989645 1loss/SRqqt2/detection_loss‚2.9179374256304333 .loss/FBt1/detection_loss‚5.124864406055874 trainer/global_step‚3099 _runtime‚1325.213413475 0loss/FRt2/assignment_loss‚3.5694958615577086 2loss/SRqqt1/assignment_loss‚1.5023600637591144 _step‚247÷’zŒ}± ‚(:’ …¥ÅÓªÛšƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚123.636' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚41.211999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0RèÆ€c² ‚(:Ø  …¥ÅÓ¢À⃠disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.drbd2041.in‚0 disk.drbd1285.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1550.out‚0) disk./.usagePercent‚32.79178468231572 disk.drbd1510.out‚0 disk.drbd1356.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 5.4609375 disk.drbd1584.out‚0 disk.drbd2078.in‚0 disk.drbd1817.out‚0 disk.sda2.in‚ 0.41015625$ disk.nvme0n1.out‚582.77783203125 disk.drbd1584.in‚0 disk.sda.out‚ 44.66796875 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.sda1.out‚0! disk.dm-0.out‚582.77783203125 disk.drbd2118.in‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd1828.out‚0 network.recv‚233777 disk.drbd2111.out‚0 cpu‚13.896636104701315 proc.cpu.threads‚29 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.drbd2041.out‚0 disk.drbd1285.in‚0 disk.drbd1510.in‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.sda2.out‚ 44.66796875 disk.drbd1356.out‚0 disk.drbd1289.in‚0$ memory_percent‚41.22788906097412 disk.drbd1938.in‚0 disk.drbd1828.in‚0# disk./.usageGB‚586.350414276123 disk.dm-0.in‚2.46875 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.drbd1965.in‚0( proc.memory.availableMB‚ 38516.890625 network.sent‚ 2.884837e+06 disk.drbd1226.out‚0* proc.memory.percent‚17.172294855117798 disk.nvme0n1.in‚2.46875 disk.sda.in‚ 0.41015625 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd2111.in‚0ÜD2³ - _wandbruntime‚1345  _runtime‚1345xù?Z´ ¼ % lr-AdamW/pg1‚8.11566152422289e-05 % lr-AdamW/pg2‚8.11566152422289e-05 trainer/global_step‚3149 % _timestamp‚1.7858116007583683e+09 _runtime‚ 1345.948313 _step‚248ÆÒèǵ Á % lr-AdamW/pg1‚8.11566152422289e-05 % lr-AdamW/pg2‚8.11566152422289e-05 trainer/global_step‚3149 % _timestamp‚1.7858116007583683e+09 _runtime‚ 1345.948313 _step‚248ø@©›2¶ - _wandbruntime‚1345  _runtime‚1345—®*…t· ¶O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2771440313344 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12403945 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2846598219264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12403947 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9024660 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33260775 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚54079829115904 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚56851286468608 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚54079846155264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2771352883712 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20930029 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 % _timestamp‚1.7858116007593017e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33260775 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚52736087485472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚55500657562896 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21428865 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚1345.952627836 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚56851181999616 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2771440313344 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2764570077424 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚54190802 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33270431 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20930027 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2764655880548 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21428605 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚54079829115904 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚56851181999616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚57054865377792 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20930027 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚54079846155264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚54208267158528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 trainer/global_step‚3149 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9024918 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚57054699314176 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚54190802 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33270431 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20930027 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2764655880548 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21428605 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚54079829115904 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚56851181999616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚57054865377792 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20930027 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚54079846155264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚54208267158528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 trainer/global_step‚3149 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9024918 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚57054699314176 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚1345.952627836 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚56851181999616ùÕ¨„2¹ - _wandbruntime‚1346  _runtime‚1346•tûÆ&º ÀM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚56869332858368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2765533353882 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2772232503296 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚57072969647104 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚56869228382208 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20936670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2772319940096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54217640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21435382 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚52752827657792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2847362607616 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33280968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54217640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20936672 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _runtime‚1346.302320361 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33271298 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2772232503296 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21435640 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20936672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12407884 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚54097012918272 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚55518378051034 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7858116011120045e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9027500 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33271298 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚3149 _step‚250 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚54225449371648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20936670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚54097012918272 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚52752844697152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54207968 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚54096995878912 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2772319940096 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚54225474275328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2847495371776 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚55518275208438 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2765447550646 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚54096995878912 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12407882 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9027756 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚57072811979264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _runtime‚1346.302320361 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33271298 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2772232503296 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21435640 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20936672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12407884 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚54097012918272 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚55518378051034 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7858116011120045e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9027500 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33271298 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚3149 _step‚250 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚54225449371648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20936670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚54097012918272 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚52752844697152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54207968 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚54096995878912 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2772319940096 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚54225474275328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2847495371776 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚55518275208438 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2765447550646 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚54096995878912 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12407882 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9027756 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚57072811979264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2765533353882 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2772232503296 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚57072969647104 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚56869228382208 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20936670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2772319940096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54217640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21435382 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚52752827657792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2847362607616 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596úI1/É2¼ - _wandbruntime‚1346  _runtime‚1346FÅoõÿ½ ù /loss/FBt1/assignment_loss‚1.009261827170849 /loss/FRt2/detection_loss‚2.6811585625012717 1loss/SRqqt1/detection_loss‚1.2107066064702419 0loss/SRqqt2/detection_loss‚3.019388784083628 trainer/global_step‚3149 1loss/SRqqt2/assignment_loss‚1.680859439796017 %loss/total_loss‚29.18387835909638 epoch‚0 0loss/FRt1/assignment_loss‚3.9616565721135744 1loss/FBt2/assignment_loss‚0.14506879853896257 % _timestamp‚1.7858116011132016e+09 /loss/FRt2/assignment_loss‚3.510929258920813 2loss/SRqqt1/assignment_loss‚1.5177466098911467 .loss/FRt1/detection_loss‚2.853984620789407 -loss/FBt1/detection_loss‚4.03038537502289 /loss/FBt2/detection_loss‚3.5627319037975775 _runtime‚1346.302977172 _step‚251gÍð¾ þ 0loss/FRt1/assignment_loss‚3.9616565721135744 1loss/FBt2/assignment_loss‚0.14506879853896257 % _timestamp‚1.7858116011132016e+09 /loss/FRt2/assignment_loss‚3.510929258920813 2loss/SRqqt1/assignment_loss‚1.5177466098911467 .loss/FRt1/detection_loss‚2.853984620789407 -loss/FBt1/detection_loss‚4.03038537502289 /loss/FBt2/detection_loss‚3.5627319037975775 _runtime‚1346.302977172 _step‚251 /loss/FBt1/assignment_loss‚1.009261827170849 /loss/FRt2/detection_loss‚2.6811585625012717 1loss/SRqqt1/detection_loss‚1.2107066064702419 0loss/SRqqt2/detection_loss‚3.019388784083628 trainer/global_step‚3149 1loss/SRqqt2/assignment_loss‚1.680859439796017 %loss/total_loss‚29.18387835909638 epoch‚0û;ºGù‘¿ ‚(:† ”¥ÅÓá‚‚ƒ gpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.746' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚42.582 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0?²ÅßcÀ ‚(:Ø  ”¥ÅÓ§…ƃ disk.drbd1828.out‚0 disk.drbd1931.out‚0 disk.rbd0.out‚0 disk.drbd1552.in‚0 disk.drbd1817.in‚0( proc.memory.availableMB‚ 38516.796875 disk.drbd2041.out‚0 disk.drbd2078.out‚0 disk.drbd1356.in‚0 disk.drbd1828.in‚0 disk.drbd2118.in‚0 disk.sda.in‚ 0.41015625 disk.sda.out‚ 45.1015625 disk.drbd2111.in‚0 disk.drbd1673.in‚0 disk.drbd1289.out‚0 disk.rbd0.in‚0 network.sent‚ 2.929831e+06 network.recv‚236257$ disk./.usageGB‚586.3509407043457 disk.drbd1965.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 5.4931640625# disk.nvme0n1.out‚583.1181640625 cpu‚13.893757205603123 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.drbd1285.in‚0 disk.sda1.in‚0 proc.cpu.threads‚29) disk./.usagePercent‚32.79181412293776 disk.drbd1510.in‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.drbd1289.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.dm-0.in‚2.46875 disk.sda2.out‚ 45.1015625 disk.nvme0n1.in‚2.46875$ memory_percent‚41.22803211212158 disk.drbd1938.out‚0 disk.dm-0.out‚583.1181640625 disk.drbd2111.out‚0 disk.drbd1510.out‚0e“iV›Á ‚(: £¥ÅÓͯ§ƒ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚118.072' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚39.35733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 /Üúc ‚(:Ø  £¥ÅÓҴʃ# disk.nvme0n1.out‚583.3525390625 disk.drbd1510.in‚0 disk.drbd2118.in‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.drbd1965.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1828.in‚0$ memory_percent‚41.22765064239502( proc.memory.availableMB‚ 38517.046875 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.drbd1817.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.drbd1828.out‚0 disk.sda.in‚ 0.41015625 disk.drbd1226.in‚0$ proc.memory.rssMB‚11254.03515625 disk.sda2.out‚ 45.43359375 disk.drbd2078.in‚0 disk.drbd2111.out‚0 disk.drbd1673.in‚0 cpu‚13.89137121108531 disk.drbd2041.in‚0 disk.dm-0.out‚583.3525390625 disk.drbd2111.in‚0 disk.drbd1552.in‚0 disk.rbd1.out‚ 5.5869140625 disk.drbd1965.out‚0 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.drbd1289.out‚0 disk.drbd1550.in‚0) disk./.usagePercent‚32.79181198955935 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.sda.out‚ 45.43359375 disk.drbd1226.out‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 network.sent‚ 2.974862e+06# disk./.usageGB‚586.350902557373 disk.rbd1.in‚0 disk.drbd1285.out‚0 disk.dm-0.in‚2.46875 proc.cpu.threads‚29 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.drbd1584.out‚0 disk.rbd0.out‚0 disk.drbd1510.out‚0 network.recv‚238819¢çÙ2à - _wandbruntime‚1366  _runtime‚1366jk+EÅÄ ¿ % lr-AdamW/pg1‚8.24452245664942e-05 % lr-AdamW/pg2‚8.24452245664942e-05 trainer/global_step‚3199 % _timestamp‚1.7858116215518787e+09 _runtime‚1366.741820766 _step‚252RPt#ÊÅ Ä % lr-AdamW/pg1‚8.24452245664942e-05 % lr-AdamW/pg2‚8.24452245664942e-05 trainer/global_step‚3199 % _timestamp‚1.7858116215518787e+09 _runtime‚1366.741820766 _step‚252ü>~&2Æ - _wandbruntime‚1366  _runtime‚1366Ô°óC»'Ç µO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚3199 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55050954 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚54938184305664 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2891836396544 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21768123 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚54938167266304 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9167593 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9167328 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚57960293331456 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚56381554642404 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 $ _timestamp‚1.785811621552792e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2815436448768 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _runtime‚1366.746634263 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12600795 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57753516285440 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚54938167266304 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2891695236608 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33798433 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚57753620754432 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚54938184305664 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2815436448768 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21262177 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21262177 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚54938167266304 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9167593 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9167328 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚57960293331456 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚56381554642404 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 $ _timestamp‚1.785811621552792e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2815436448768 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _runtime‚1366.746634263 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12600795 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57753516285440 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚54938167266304 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2891695236608 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33798433 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚57753620754432 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚54938184305664 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2815436448768 IDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21268822 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚54955334029312 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _step‚254 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚54955334029312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚54955351068672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚57978408804352 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2809421817390 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55068120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21268820 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12604732 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12604734 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9170145 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55077792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocatv˜„ôëed‚56399275130542 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚1367.093737629 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55077792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33799300 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21774877 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚56399172287946 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 $ _timestamp‚1.785811621903452e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55068120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2816316075520 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9170383 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2892603592704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2816228638720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33808970 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33799300 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55085830115328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21268820 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33808970 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 trainer/global_step‚3199 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚57771667144192 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚54955351068672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21775117 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55085805211648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2809336014154ÓcÔÊ&Ë ÄM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚54955334029312 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _step‚254 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚54955334029312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚54955351068672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚57978408804352 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2809421817390 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55068120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21268820 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12604732 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12604734 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9170145 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55077792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚56399275130542 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚1367.093737629 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55077792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33799300 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21774877 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚56399172287946 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 $ _timestamp‚1.785811621903452e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55068120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2816316075520 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9170383 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2892603592704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2816228638720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33808970 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33799300 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55085830115328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21268820 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33808970 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 trainer/global_step‚3199 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚57771667144192 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚54955351068672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21775117 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55085805211648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2809336014154 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21268822 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚57771667144192 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21268822 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251þ—@`p2Ì - _wandbruntime‚1367  _runtime‚1367ÛåMÍ þ /loss/FRt1/assignment_loss‚4.429945689301158 .loss/FRt2/detection_loss‚2.736809350293258 epoch‚0 0loss/FRt2/assignment_loss‚3.9102321986494393 1loss/FBt2/assignment_loss‚0.06054853492942466 _runtime‚1367.094438586 2loss/SRqqt2/assignment_loss‚1.5440641944868523 /loss/FRt1/detection_loss‚2.4961706581850382 /loss/FBt2/detection_loss‚3.1384051982448473 &loss/total_loss‚30.389693042619356 % _timestamp‚1.7858116219046462e+09 2loss/SRqqt1/assignment_loss‚1.6461864491035272 0loss/FBt1/assignment_loss‚0.3254819754511118 1loss/SRqqt1/detection_loss‚1.0672734123262282 1loss/SRqqt2/detection_loss‚3.1408849280131492 .loss/FBt1/detection_loss‚5.893690453635322 trainer/global_step‚3199 _step‚255Ûáþ³ Î ƒ 2loss/SRqqt2/assignment_loss‚1.5440641944868523 /loss/FRt1/detection_loss‚2.4961706581850382 /loss/FBt2/detection_loss‚3.1384051982448473 &loss/total_loss‚30.389693042619356 % _timestamp‚1.7858116219046462e+09 2loss/SRqqt1/assignment_loss‚1.6461864491035272 0loss/FBt1/assignment_loss‚0.3254819754511118 1loss/SRqqt1/detection_loss‚1.0672734123262282 1loss/SRqqt2/detection_loss‚3.1408849280131492 .loss/FBt1/detection_loss‚5.893690453635322 trainer/global_step‚3199 _step‚255 /loss/FRt1/assignment_loss‚4.429945689301158 .loss/FRt2/detection_loss‚2.736809350293258 epoch‚0 0loss/FRt2/assignment_loss‚3.9102321986494393 1loss/FBt2/assignment_loss‚0.06054853492942466 _runtime‚1367.094438586ÿ™:£Y‘Ï ‚(:† ²¥ÅÓÑëò‚ gpu.0.gpu‚37.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚119.196' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚39.732 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0<¢o_Ð ‚(:Ô  ²¥ÅÓÄÈÁƒ network.sent‚ 2.977893e+06 disk.nvme0n1.in‚2.46875# disk.nvme0n1.out‚583.7724609375 disk.drbd2078.in‚0 disk.drbd2078.out‚0 cpu‚13.8892967897701 disk.drbd2111.out‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.drbd1931.in‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.drbd1552.in‚0 disk.sda2.out‚ 45.83984375$ proc.memory.rssMB‚11254.03515625 disk.sda.out‚ 45.83984375 disk.drbd1828.out‚0 disk.drbd2111.in‚0$ memory_percent‚41.22810363769531 proc.cpu.threads‚29 network.recv‚239719 disk.drbd1356.in‚0 disk.rbd0.out‚0 disk.drbd1510.out‚0) disk./.usagePercent‚32.79181305624855 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1285.out‚0 disk.dm-0.in‚2.46875$ disk./.usageGB‚586.3509216308594 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd1965.in‚0 disk.dm-0.out‚583.7724609375$ proc.memory.availableMB‚38516.75 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.in‚0 disk.drbd1289.in‚0 disk.drbd1938.out‚0 disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.sda.in‚ 0.41015625 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 5.6494140625 disk.drbd1356.out‚0 disk.drbd1965.out‚0 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0* proc.memory.percent‚17.172294855117798 disk.drbd1226.in‚0VIßÛ2Ñ - _wandbruntime‚1387  _runtime‚1387—R£AÆÒ À & lr-AdamW/pg1‚8.373383389075951e-05 & lr-AdamW/pg2‚8.373383389075951e-05 trainer/global_step‚3249 $ _timestamp‚1.785811642363585e+09 _runtime‚1387.553542581 _step‚256ñ¢•ËÓ Å & lr-AdamW/pg1‚8.373383389075951e-05 & lr-AdamW/pg2‚8.373383389075951e-05 trainer/global_step‚3249 $ _timestamp‚1.785811642363585e+09 _runtime‚1387.553542581 _step‚256€ÊÐt2Ô - _wandbruntime‚1387  _runtime‚1387èM*8¼'Õ ¶O SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9309394 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21594327 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2936875701760 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2859345154560 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21594329 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34326435 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚57262451721912 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚54410104717472 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21594329 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2859432584192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚3249 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34316779 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚58655955040256 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚55796505416704 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2859345154560 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9309650 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚57262554564396 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚58655850571264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7858116423645208e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34316779 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚58865829636608 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚55796522456064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22107039 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚58655955040256 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34326435 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12797647 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2852432807564 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22107297 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _runtime‚1387.557651762 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚58655850571264 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚55796522456064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2859432584192 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚55928978838528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55911106 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2937010570240 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚54410121756832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121f¢Wp Ö »O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34316779 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚58865829636608 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚55796522456064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22107039 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚58655955040256 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34326435 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12797647 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2852432807564 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22107297 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _runtime‚1387.557651762 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚58655850571264 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚55796522456064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2859432584192 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚55928978838528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55911106 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.largeÜÞ¯…±_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2937010570240 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚54410121756832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9309394 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21594327 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2936875701760 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2859345154560 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21594329 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34326435 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚57262451721912 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚54410104717472 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21594329 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2859432584192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚3249 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34316779 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚58655955040256 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚55796505416704 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2859345154560 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9309650 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚57262554564396 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚58655850571264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7858116423645208e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538¾§íN2× - _wandbruntime‚1387  _runtime‚1387按tÅ&Ø ¿M C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚58674001430016 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚55813689219072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2860312210944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22114041 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34336972 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚58674001430016 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚55813689219072 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55937944 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55937944 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34336972 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚57280172210050 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚54426861929152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚55813672179712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2937786945536 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 % _timestamp‚1.7858116427184982e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34327302 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9312457 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55946161051648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12801582 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21600970 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55928272 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2937911321088 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21600972 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚58884097276416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 _runtime‚ 1387.90881034 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 trainer/global_step‚3249 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2860224774144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55946185955328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚54426844889792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34327302 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚57280069367454 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22113795 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21600972 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚58673896953856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670̽ieÊ&Ù ÄM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚58884097276416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 _runtime‚ 1387.90881034 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 trainer/global_step‚3249 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2860224774144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55946185955328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚54426844889792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34327302 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚57280069367454 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22113795 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21600972 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚58673896953856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚58674001430016 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚55813689219072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2860312210944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22114041 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34336972 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚58674001430016 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚55813689219072 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55937944 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55937944 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34336972 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚57280172210050 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚54426861929152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚55813672179712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2937786945536 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 % _timestamp‚1.7858116427184982e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34327302 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9312457 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55946161051648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12801582 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21600970 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55928272 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2937911321088 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21600972 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21926477 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56780916 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21926477 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚56789309774848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2982056979968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚56771258 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚56654860606464 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚56789334678528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 @:DeviceStatsMoniª„À tor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22447218 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚59771522332672 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚261 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚59558289326080 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2896321271072 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34844781 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚56654860606464 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2982187654144 trainer/global_step‚3299 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34854437 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34854437 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12994495 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚55247113333472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚58143348801420 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚55247130372832 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2903428719616 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚59558184857088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12994497 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2903341289984 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚59558289326080 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2903428719616 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2903341289984 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9452974 % _timestamp‚1.7858116633064158e+09 _runtime‚1408.499626625ÿÓíxÁ'ä »O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12994495 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚55247113333472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚58143348801420 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚55247130372832 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2903428719616 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚59558184857088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12994497 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2903341289984 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚59558289326080 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2903428719616 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2903341289984 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9452974 % _timestamp‚1.7858116633064158e+09 _runtime‚1408.499626625 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21926477 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56780916 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21926477 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚56789309774848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2982056979968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚56771258 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚56654860606464 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚56789334678528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22447218 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚59771522332672 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚261 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚59558289326080 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2896321271072 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34844781 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚56654860606464 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2982187654144 trainer/global_step‚3299 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34854437 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34854437 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392…[˜2å - _wandbruntime‚1408  _runtime‚1408ÛÆ&æ ÀM O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56788424 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚56672027369472 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚55263853505792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56798096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21933120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22454194 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21933120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7858116636484618e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21933122 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2982960027648 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2904220909568 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚56672027369472 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚59789634587136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2897112941170 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚56672010330112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚59576335715840 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚59576231239680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34864974 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2904308346368 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2904220909568 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22453944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2904308346368 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34855304 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚56806516891648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56798096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚55263853505792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56798096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21933120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22454194 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21933120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7858116636484618e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21933122 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2982960027648 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2904220909568 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚56672027369472 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚59789634587136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2897112941170 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚56672010330112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚59576335715840 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚59576231239680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34864974 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceÈîRgStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2904308346368 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2904220909568 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22453944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2904308346368 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34855304 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚56806516891648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56798096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56788424 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚56672027369472 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2947337425408 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9595710 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56084121949472 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚57513181717504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3027371813888 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2940209734580 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚57649690518528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57631410 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚60460519142912 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22787057 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2940123931456 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2947424855040 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35382439 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚60460623611904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2947424855040 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚60460623611904 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22258629 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22258627 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚60460519142912 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚60676900463104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35382439 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2947337425408 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9595710 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56084121949472 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚57513181717504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3027371813888 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2940209734580 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚57649690518528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57631410 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚60460519142912 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22787057 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2940123931456 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2947424855040 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35382439 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚60460623611904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2947424855040 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚60460623611904 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22258629 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22258627 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚60460519142912 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚60676900463104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35382439 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚57666897635328 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22793812 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚59041966369066 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1429.184473361 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚57658248 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22265270 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3028136241152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚60478565525504 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56100879161152 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13195282 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚60478670001664 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 $ _timestamp‚1.785811683994193e+09 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚60695166640640 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35383306 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57658248 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2948217044992 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35383306 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚60695008972800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57648576 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2941087207914 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚3349 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9598528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2948217044992 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚57530365519872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22265272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9598273 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚59041863526470 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚57530348480512 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22265270 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3028269005312 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2941001404678 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22265272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22793555 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35392976 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚56100862121792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57648576 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2948304481792 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35392976 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚60478670001664 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚57666872731648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130+£™nÊ&÷ ÄM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚57666897635328 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22793812 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚59041966369066 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1429.184473361 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚57658248 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22265270 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3028136241152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚60478565525504 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56100879161152 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13195282 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚60478670001664 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 $ _timestamp‚1.785811683994193e+09 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚60695166640640 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35383306 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57658248 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2948217044992 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35383306 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚60695008972800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57648576 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2941087207914 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚3349 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9598528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2948217044992 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚57530365519872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22265272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9598273 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚59041863526470 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚57530348480512 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22265270 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3028269005312 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2941001404678 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22265272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22793555 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35392976 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚56100862121792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57648576 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2948304481792 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35392976 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚60478670001664 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚57666872731648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0ŠÄaº2ø - _wandbruntime‚1429  _runtime‚1429Ç=øù ÿ 1loss/FBt1/assignment_loss‚0.18166495344485156 1loss/FBt2/assignment_loss‚0.11589099865987561 1loss/SRqqt1/detection_loss‚0.9027723382697337 /loss/FBt1/detection_loss‚3.8399750516499807 &loss/total_loss‚24.908174353817714 % _timestamp‚1.7858116839953988e+09 _step‚267 .loss/FRt1/detection_loss‚2.574036799842798 .loss/FRt2/detection_loss‚2.374718709259617 _runtime‚1429.185194135 0loss/FRt2/assignment_loss‚3.7005452112275727 2loss/SRqqt1/assignment_loss‚1.4672889733120151 2loss/SRqqt2/assignment_loss‚1.8382731330773188 trainer/global_step‚3349 /loss/FRt1/assignment_loss‚3.829436780531195 1loss/SRqqt2/detection_loss‚2.5540468716432176 /loss/FBt2/detection_loss‚1.5295245328995346 epoch‚0R4û ú „ 0loss/FRt2/assignment_loss‚3.7005452112275727 2loss/SRqqt1/assignment_loss‚1.4672889733120151 2loss/SRqqt2/assignment_loss‚1.8382731330773188 trainer/global_step‚3349 /loss/FRt1/assignment_loss‚3.829436780531195 1loss/SRqqt2/detection_loss‚2.5540468716432176 /loss/FBt2/detection_loss‚1.5295245328995346 epoch‚0 1loss/FBt1/assignment_loss‚0.18166495344485156 1loss/FBt2/assignment_loss‚0.11589099865987561 1loss/SRqqt1/detection_loss‚0.9027723382697337 /loss/FBt1/detection_loss‚3.8399750516499807 &loss/total_loss‚24.908174353817714 % _timestamp‚1.7858116839953988e+09 _step‚267 .loss/FRt1/detection_loss‚2.574036799842798 .loss/FRt2/detection_loss‚2.374718709259617 _runtime‚1429.185194135‹Z½‘û ‚(:† î¥ÅÓ‹Øë‚ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚119.853' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚39.951 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 @phü ‚(:Ý  î¥ÅÓÆæ°ƒ disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd1828.out‚0 disk.drbd2118.out‚0* proc.memory.availableMB‚38515.95703125 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0) disk./.usagePercent‚32.79181220289719 disk.sda.out‚ 47.43359375 disk.drbd1965.out‚0 disk.drbd1510.in‚0$ memory_percent‚41.22931361198425 disk.drbd1938.in‚0$ disk.nvme0n1.out‚584.78076171875 disk.drbd1584.out‚0 disk.drbd1550.in‚0 disk.rbd1.out‚ 5.837890625 disk.drbd1584.in‚0* proc.memory.percent‚17.172294855117798 disk.drbd2118.in‚0 disk.drbd1226.out‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 0.41015625 disk.drbd1285.out‚0! disk.dm-0.out‚584.78076171875 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.drbd1931.in‚0 network.sent‚ 3.116146e+06 disk.drbd1931.out‚0 disk.drbd2078.in‚0 disk.dm-0.in‚2.46875 network.recv‚247874 disk.drbd1828.in‚0 disk.drbd2111.in‚0 proc.cpu.threads‚29 cpu‚13.880495407804547 disk.drbd1552.in‚0 disk.sda2.out‚ 47.43359375 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.drbd1226.in‚0$ disk./.usageGB‚586.3509063720703 disk.sda.in‚ 0.41015625 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd2041.in‚0 disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0$ proc.memory.rssMB‚11254.03515625 ­Å2ý - _wandbruntime‚1449  _runtime‚1449p Çþ Á % _timestamp‚1.7858117045022695e+09 _runtime‚1449.692206196 _step‚268 & lr-AdamW/pg1‚8.759966186355543e-05 & lr-AdamW/pg2‚8.759966186355543e-05 trainer/global_step‚3399WM§dÌÿ Æ & lr-AdamW/pg1‚8.759966186355543e-05 & lr-AdamW/pg2‚8.759966186355543e-05 trainer/global_step‚3399 % _timestamp‚1.7858117045022695e+09 _runtime‚1449.692206196 _step‚268Œøî€2€ - _wandbruntime‚1449  _runtime‚1449=v~ìT# ¶O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2991420990464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚261 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22590779 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚61582564283904 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35910441 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9738297 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2991333560832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚58371519867904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚56921147604832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3072517925376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚58510021454848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3072385154048 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22590777 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚61362957897728 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13388197 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚61362957897728 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚58371519867904 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚59905142960436 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚58491562 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚58501220 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35900785 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22590779 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23126753 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚259 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2991420990464 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚61362853428736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13388195 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 trainer/global_step‚3399 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23126492 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56921130565472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XIDeviceStatsMonitor.on_train_batch_start/rð[oheserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚58371536907264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚1449.696655744 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚58510046358528 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2991333560832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251(\AÁ'‚ »O N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚1449.696655744 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚58510046358528 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2991333560832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2991420990464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚261 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22590779 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚61582564283904 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35910441 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9738297 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2991333560832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚58371519867904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚56921147604832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3072517925376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚58510021454848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3072385154048 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22590777 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚61362957897728 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13388197 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚61362957897728 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚58371519867904 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚59905142960436 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚58491562 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚58501220 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35900785 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22590779 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23126753 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚259 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2991420990464 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚61362853428736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13388195 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 trainer/global_step‚3399 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23126492 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56921130565472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚58371536907264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548ôhÖÍ2ƒ - _wandbruntime‚1450  _runtime‚1450m¡B§Æ&„ ÀM 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58518400 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2992213180416 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2984975671422 _step‚270 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚58527228571648 % _timestamp‚1.7858117048524377e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚58388703670272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22597420 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13392132 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚58388686630912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3073418393600 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22597420 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56937887777152 _runtime‚1450.042699093 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35920978 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚59922863448574 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23133251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13392134 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚56937870737792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚58388686630912 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58518400 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2984889868186 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚61380899811328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚61381004287488 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9741119 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35911308 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚58508728 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2992300617216 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚58388703670272 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚61600516298240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3073287726592 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2992213180416 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35920978 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22597422 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35911308 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253^…¯Ë&… ÅM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2984889868186 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚61380899811328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚61381004287488 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9741119 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35911308 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚58508728 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2992300617216 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚58388703670272 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚61600516298240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3073287726592 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2992213180416 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35920978 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22597422 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35911308 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58518400 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2992213180416 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2984975671422 _step‚270 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚58527228571648 % _timestamp‚1.7858117048524377e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚58388703670272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22597420 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13392132 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚58388686630912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3073418393600 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22597420 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56937887777152 _runtime‚1450.042699093 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35920978 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚59922863448574 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23133251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13392134 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚56937870737792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚58388686630912 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58518400 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251Žhñ}2† - _wandbruntime‚1450  _runtime‚1450õKDć þ 0loss/FRt2/assignment_loss‚3.5229493037308797 1loss/FBt2/assignment_loss‚0.06450968020339459 .loss/FBt1/detection_loss‚6.500381781802616 _runtime‚1450.043396763 2loss/SRqqt1/assignment_loss‚1.4795968516352977 .loss/FRt1/detection_loss‚2.569911252707243 $ _timestamp‚1.785811704853629e+09 _step‚271 0loss/FRt1/assignment_loss‚3.9241253793239594 2loss/SRqqt2/assignment_loss‚1.5731239835190218 /loss/FRt2/detection_loss‚2.4973538252386716 0loss/SRqqt2/detection_loss‚2.198699411611224 /loss/FBt2/detection_loss‚1.9053299506163903 &loss/total_loss‚27.424877385245978 trainer/global_step‚3399 1loss/FBt1/assignment_loss‚0.22767340822611004 1loss/SRqqt1/detection_loss‚0.9612225566311711 epoch‚0x’ p¦ˆ ƒ _step‚271 0loss/FRt1/assignment_loss‚3.9241253793239594 2loss/SRqqt2/assignment_loss‚1.5731239835190218 /loss/FRt2/detection_loss‚2.4973538252386716 0loss/SRqqt2/detection_loss‚2.198699411611224 /loss/FBt2/detection_loss‚1.9053299506163903 &loss/total_loss‚27.424877385245978 trainer/global_step‚3399 1loss/FBt1/assignment_loss‚0.22767340822611004 1loss/SRqqt1/detection_loss‚0.9612225566311711 epoch‚0 0loss/FRt2/assignment_loss‚3.5229493037308797 1loss/FBt2/assignment_loss‚0.06450968020339459 .loss/FBt1/detection_loss‚6.500381781802616 _runtime‚1450.043396763 2loss/SRqqt1/assignment_loss‚1.47959685•ªxc16352977 .loss/FRt1/detection_loss‚2.569911252707243 $ _timestamp‚1.785811704853629e+09ÕF$œ‰ ‚(:‘ ý¥ÅÓÍý̃ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚126.971' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.32366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‘µ’ÊgŠ ‚(:Ü  ý¥ÅÓþšÝƒ disk.drbd1817.out‚0 disk.drbd1828.out‚0 proc.cpu.threads‚29 cpu‚13.878869974877894 network.sent‚ 3.161092e+06 disk.drbd1817.in‚0 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.drbd1552.out‚0 disk.drbd1356.out‚0 disk.sda.out‚ 47.6796875 disk.drbd1550.out‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.drbd2078.in‚0% memory_percent‚41.229575872421265 disk.drbd1289.in‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 47.6796875 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0$ disk.nvme0n1.out‚584.96044921875 disk.drbd1828.in‚0 disk.dm-0.in‚2.46875 disk.sda1.out‚0* proc.memory.availableMB‚38515.78515625 network.recv‚250413) disk./.usagePercent‚32.79181220289719 disk.drbd1584.out‚0 disk.drbd1285.out‚0! disk.dm-0.out‚584.96044921875 disk.drbd2118.in‚0 disk.drbd1356.in‚0 disk.sda.in‚ 0.41015625 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1931.in‚0* proc.memory.percent‚17.172294855117798 disk.rbd1.out‚ 5.837890625 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.drbd1673.out‚0 disk.drbd1938.out‚0 disk.drbd1965.in‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 disk.sda1.in‚0 disk.drbd2041.out‚0 disk.sda2.in‚ 0.41015625$ disk./.usageGB‚586.3509063720703 disk.drbd1965.out‚0$ proc.memory.rssMB‚11254.03515625Qˆ‹ ‚(:’ Œ¦ÅÓ÷ƒ¨ƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚122.702' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚40.900666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0cÕ¤éVŒ ‚(:Ë  Œ¦ÅÓÒÀµƒ* proc.memory.percent‚17.172294855117798 disk.sda.out‚48 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.drbd2118.out‚0 disk.drbd1226.out‚0 disk.drbd2111.in‚0 disk.sda2.out‚48) proc.memory.availableMB‚ 38515.7890625 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.drbd1510.in‚0 disk.drbd2078.out‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0) disk./.usagePercent‚32.79180132266731 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd1550.out‚0 disk.drbd1289.out‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 network.sent‚ 3.206088e+06 disk.drbd1226.in‚0 disk.drbd1285.in‚0 disk.sda1.out‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 network.recv‚252647$ disk./.usageGB‚586.3507118225098 disk.drbd1552.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd2041.in‚0 disk.sda.in‚ 0.41015625 disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.rbd1.out‚ 5.8701171875 disk.drbd1356.out‚0 disk.drbd1584.out‚0$ memory_percent‚41.22956991195679! disk.dm-0.out‚585.19970703125 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd1289.in‚0$ disk.nvme0n1.out‚585.19970703125 proc.cpu.threads‚29 disk.drbd1285.out‚0 disk.drbd1817.out‚0 disk.drbd1938.out‚0 disk.drbd2078.in‚0 disk.drbd2111.out‚0 disk.drbd2118.in‚0$ proc.memory.rssMB‚11254.03515625 cpu‚13.876636506801017 disk.drbd1673.out‚0 disk.drbd2041.out‚0ýˆñ/2 - _wandbruntime‚1470  _runtime‚1470¬q˜xÆŽ À & lr-AdamW/pg2‚8.888827118782073e-05 trainer/global_step‚3449 $ _timestamp‚1.785811725067207e+09 _runtime‚1470.257150825 _step‚272 & lr-AdamW/pg1‚8.888827118782073e-05`ï*Ë Å & lr-AdamW/pg1‚8.888827118782073e-05 & lr-AdamW/pg2‚8.888827118782073e-05 trainer/global_step‚3449 $ _timestamp‚1.785811725067207e+09 _runtime‚1470.257150825 _step‚272 É2 - _wandbruntime‚1470  _runtime‚1470’gZs»'‘ µO TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚60786040039944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59351714 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚62487942621696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚62265292183552 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3035417125888 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚62265187714560 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22922927 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36438443 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22922929 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3027986661596 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36428787 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚59361372 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚62265187714560 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚59229875057664 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚59229858018304 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0vWÀ'’ ºO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚62265187714560 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22922927 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36438443 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22922929 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3027986661596 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36428787 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚59361372 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚62265187714560 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚59229875057664 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚59229858018304 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚60786040039944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59351714 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚62487942621696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚62265292183552 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3035417125888 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚62283338573312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚59387584411648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23473108 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13588982 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚60803657685486 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22929570 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3028864134930 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59368880 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22929572 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36448980 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.curre ë nt‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36439310 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9884378 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3028778331694 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3036209315840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59378552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚59247024781312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3036296752640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚274 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚62506049282048 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130­æõ Ê&• ÄM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23473362 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚59387609315328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚59378552 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 trainer/global_step‚3449 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13588984 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23473108 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13588982 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚60803657685486 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22929570 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3028864134930 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59368880 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22929572 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36448980 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36439310 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9884378 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3028778331694 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3036209315840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59378552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚59247024781312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3036296752640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚274 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚62506049282048 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚59368880 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9884126 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3036209315840 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36439310 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚62506209047040 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22929570 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3036296752640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3118599731712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚62283338573312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚59387584411648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚63167522000384 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13781897 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3079413261312 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3162684261888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚63167626469376 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚60211866 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 trainer/global_step‚3499 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚60088213208064 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23806494 $ _timestamp‚1.785811745818643e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚60088196168704 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60221524 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚58595147797472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13781895 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚63393417396736 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚61667039961936 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚63167522000384 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚63167626469376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10024352 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3079413261312 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3079325831680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚63393572974592 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291$ ö8Ê  ¼O cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3162814936064 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚58595147797472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13781895 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚63393417396736 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚61667039961936 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚63167522000384 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚63167626469376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10024352 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3079413261312 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3079325831680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚63393572974592 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36956789 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚63167522000384 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13781897 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3079413261312 _MDeviceStatsMonitor.on_train_batch©q¼ø_start/inactive_split_bytes.small_pool.freed‚ 3162684261888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚63167626469376 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚60211866 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 trainer/global_step‚3499 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚60088213208064 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23806494 $ _timestamp‚1.785811745818643e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚60088196168704 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60221524 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚278 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36967312 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23261720 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3080205451264 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60238704 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36967312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚60247940251648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3163722334720 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13785834 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60238704 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61684657607590 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23261720 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚60105379971072 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚60105379971072 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10027364 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3072666795202 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚60229032 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚63185672859136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚1491.363174424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3080292888064 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚63411687490048 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚60229032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23812948 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13785832 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3072752598438 trainer/global_step‚3499 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚60105362931712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23813198 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23261722 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36976982 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚63411523530752 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚63185672859136 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23812948 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13785832 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3072752598438 trainer/global_step‚3499 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚60105362931712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23813198 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23261722 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36976982 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚63411523530752 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚63185672859136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚278 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36967312 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23261720 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3080205451264 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60238704 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36967312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚60247940251648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3163722334720 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13785834 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60238704 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61684657607590 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23261720 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚60105379971072 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚60105379971072 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10027364 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3072666795202 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚60229032 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚63185672859136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚1491.363174424–¥^l´2¤ - _wandbruntime‚1491  _runtime‚1491“~^¥ ü .loss/FRt2/detection_loss‚2.503442582876786 .loss/FBt1/detection_loss‚5.006825055394854 _runtime‚1491.364189069 /loss/FRt2/assignment_loss‚3.721136963885763 1loss/FBt2/assignment_loss‚0.19596935577286395 1loss/SRqqt2/detection_loss‚2.2487822912136712 1loss/SRqqt1/assignment_loss‚1.604992440443928 1loss/SRqqt1/detection_loss‚0.9612869864853567 epoch‚0 trainer/global_step‚3499 % _timestamp‚1.7858117461743975e+09 0loss/FRt1/assignment_loss‚4.2146175542435085 0loss/FBt1/assignment_loss‚0.3020085300896816 .loss/FBt2/detection_loss‚2.112069989673078 &loss/total_loss‚27.037554561846733 _step‚279 2loss/SRqqt2/assignment_loss‚1.5588034971555074 /loss/FRt1/detection_loss‚2.6076193146117324Ul¦  1loss/SRqqt1/assignment_loss‚1.604992440443928 1loss/SRqqt1/detection_loss‚0.9612869864853567 epoch‚0 trainer/global_step‚3499 % _timestamp‚1.7858117461743975e+09 0loss/FRt1/assignment_loss‚4.2146175542435085 0loss/FBt1/assignment_loss‚0.3020085300896816 .loss/FBt2/detection_loss‚2.112069989673078 &loss/total_loss‚27.037554561846733 _step‚279 2loss/SRqqt2/assignment_loss‚1.5588034971555074 /loss/FRt1/detection_loss‚2.6076193146117324 .loss/FRt2/detection_loss‚2.503442582876786 .loss/FBt1/detection_loss‚5.006825055394854 _runtime‚1491.364189069 /loss/FRt2/assignment_loss‚3.721136963885763 1loss/FBt2/assignment_loss‚0.19596935577286395 1loss/SRqqt2/detection_loss‚2.2487822912136712—®#åÿœ§ ‚(:‘ ª¦ÅÓˆýû‚ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚124.133' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.37766666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¼Ãµg¨ ‚(:Ü  ª¦ÅÓõƒ¿ƒ disk.drbd1552.in‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd1931.out‚0) disk./.usagePercent‚32.79185337710047 disk.drbd1226.out‚0 disk.drbd1510.in‚0 disk.sda1.out‚0 network.sent‚ 3.254059e+06 disk.drbd2111.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.dm-0.in‚2.46875 disk.drbd1289.out‚0 disk.sda2.in‚ 0.41015625 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1938.out‚0 cpu‚13.872365222788673 network.recv‚256007$ disk./.usageGB‚586.3516426086426 disk.drbd2118.in‚0 disk.drbd2118.out‚0* proc.memory.availableMB‚38515.45703125 disk.drbd2078.out‚0 disk.dm-0.out‚585.7236328125 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.sda1.in‚0 disk.drbd1931.in‚0 disk.sda.out‚ 48.84765625 disk.rbd0.out‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1552.out‚0 disk.sda2.out‚ 48.84765625 disk.sda.in‚ 0.41015625 disk.drbd1285.out‚0 disk.drbd1817.out‚0# disk.nvme0n1.out‚585.7236328125 disk.drbd2041.out‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd1673.out‚0* proc.memory.percent‚17.172294855117798$ memory_percent‚41.23007655143738 disk.drbd2041.in‚0 disk.rbd0.in‚0 disk.drbd1356.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1965.in‚0 disk.rbd1.out‚ 6.1123046875 proc.cpu.threads‚29Þ;A2© - _wandbruntime‚1512  _runtime‚1512†åïǪ Á % _timestamp‚1.7858117668403506e+09 _runtime‚1512.030288314 _step‚280 & lr-AdamW/pg1‚9.146548983635135e-05 & lr-AdamW/pg2‚9.146548983635135e-05 trainer/global_step‚3549ê¸mßÌ« Æ _runtime‚1512.030288314 _step‚280 & lr-AdamW/pg1‚9.146548983635135e-05 & lr-AdamW/pg2‚9.146548983635135e-05 trainer/global_step‚3549 % _timestamp‚1.7858117668403506e+09˜B¢êØ2¬ - _wandbruntime‚1512  _runtime‚1512ÿkשּׂ ¸O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3207950965248 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚59432156413472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3123321967104 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23587227 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10167881 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3123409396736 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37484791 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚61091088974848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚60946534319104 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37494447 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚64069960755200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚64069856286208 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23587227 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocateQ“Ĥd‚61091113878528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37494447 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24146628 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚60946534319104 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10167626 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3115677785488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚64298905071616 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚62547937041444 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _step‚281 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 trainer/global_step‚3549 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61072018 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61081676 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3123321967104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚64069960755200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23587229 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚60946551358464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13978747 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚64299064843776 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37484791 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13978745 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚1512.034621369»QðþÃ'® ½O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23587229 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚60946551358464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13978747 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚64299064843776 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37484791 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13978745 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚1512.034621369 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3207950965248 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚59432156413472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3123321967104 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23587227 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10167881 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3123409396736 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37484791 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚61091088974848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚60946534319104 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37494447 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚64069960755200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚64069856286208 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23587227 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚61091113878528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37494447 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24146628 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚60946534319104 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10167626 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3115677785488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚64298905071616 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚62547937041444 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _step‚281 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 trainer/global_step‚3549 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61072018 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61081676 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3123321967104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚64069960755200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888™6Õs’2¯ - _wandbruntime‚1512  _runtime‚1512äE*È&° ÂM >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚59448896585792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚60963718121472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3124201586688 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚64317176221696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚62565451844502 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23593870 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24153405 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13982684 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _step‚282 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23593872 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37495314 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3124201586688 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37504984 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚60963701082112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚265 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61098856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61108296091648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚59448913625152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10170458 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚64088007144960 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚64088007144960 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61089184 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 141152768 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚62565554687098 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23593872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3208855226368 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7858117671899312e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚3549 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚60963701082112 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3124289023488 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24153140 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13982682 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3208714073600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37504984 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚61108320995328 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3124289023488 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61089184 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚64087902668800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37495314 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23593870 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 _runtime‚1512.380203013 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61098856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61108296091648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚59448913625152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10170458 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚64088007144960 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚64088007144960 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61089184 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 141152768 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚62565554687098 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23593872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3208855226368 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7858117671899312e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚3549 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚60963701082112 OADeviceStatsMonitor.on_train_batch_end/reåÕ a¿ quested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3124289023488 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24153140 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13982682 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3208714073600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37504984 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚61108320995328 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3124289023488 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61089184 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚64087902668800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37495314 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23593870 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 _runtime‚1512.380203013 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚59448896585792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0šÉý€t2² - _wandbruntime‚1512  _runtime‚1512«"dz ú /loss/FRt1/assignment_loss‚3.788341638204214 .loss/FRt1/detection_loss‚2.622887394315488 /loss/FRt2/detection_loss‚2.5337056877712407 1loss/SRqqt1/detection_loss‚1.0067821786420652 /loss/FBt2/detection_loss‚2.8580245608247252 trainer/global_step‚3549 _runtime‚ 1512.38094822 2loss/SRqqt1/assignment_loss‚1.4543992715942509 1loss/FBt2/assignment_loss‚0.03932295390126996 0loss/SRqqt2/detection_loss‚2.297294537388328 $ _timestamp‚1.785811767191137e+09 1loss/SRqqt2/assignment_loss‚1.647851598435554 0loss/FBt1/assignment_loss‚0.3326826292193598 .loss/FBt1/detection_loss‚5.031972547372182 epoch‚0 _step‚283 0loss/FRt2/assignment_loss‚4.0283001984159155 &loss/total_loss‚27.641565196084592âJ±´ ÿ 1loss/SRqqt2/assignment_loss‚1.647851598435554 0loss/FBt1/assignment_loss‚0.3326826292193598 .loss/FBt1/detection_loss‚5.031972547372182 epoch‚0 _step‚283 0loss/FRt2/assignment_loss‚4.0283001984159155 &loss/total_loss‚27.641565196084592 /loss/FRt1/assignment_loss‚3.788341638204214 .loss/FRt1/detection_loss‚2.622887394315488 /loss/FRt2/detection_loss‚2.5337056877712407 1loss/SRqqt1/detection_loss‚1.0067821786420652 /loss/FBt2/detection_loss‚2.8580245608247252 trainer/global_step‚3549 _runtime‚ 1512.38094822 2loss/SRqqt1/assignment_loss‚1.4543992715942509 1loss/FBt2/assignment_loss‚0.03932295390126996 0loss/SRqqt2/detection_loss‚2.297294537388328 $ _timestamp‚1.785811767191137e+09›XûN‘µ ‚(:† ¹¦ÅÓ‹Ô‚ƒ gpu.0.gpu‚36.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚129.333' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.111 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ì/ÿ [¶ ‚(:Ð  ¹¦ÅÓ†žÐƒ$ disk./.usageGB‚586.3516426086426 disk.drbd1673.in‚0 disk.rbd1.out‚ 6.1123046875 disk.drbd1226.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 disk.drbd1673.out‚0 disk.drbd1828.in‚0 disk.dm-0.in‚2.46875 disk.dm-0.out‚ 586.23046875% proc.memory.availableMB‚ 38515.375 disk.sda1.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1510.in‚0 disk.drbd1965.out‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.drbd1584.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1285.out‚0 disk.drbd2041.out‚0 disk.sda.out‚ 49.171875 disk.drbd1510.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.rbd1.in‚0 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd2078.out‚0 cpu‚13.870328367899676 network.recv‚258395) disk./.usagePercent‚32.79185337710047 disk.drbd1289.in‚0 disk.drbd1931.in‚0 disk.drbd1550.in‚0 network.sent‚ 3.299017e+06% memory_percent‚41.230201721191406 disk.drbd1817.in‚0! disk.nvme0n1.out‚ 586.23046875 disk.drbd1226.out‚0 disk.drbd2111.out‚0 disk.drbd1817.out‚0 disk.drbd1931.out‚0 proc.cpu.threads‚29 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1584.out‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd2118.out‚0 disk.sda2.out‚ 49.171875 disk.drbd2041.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.sda1.out‚0eÒ´j· ‚(:’ ȦÅÓÚü¦ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚113.932' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.977333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ð„Ð?h¸ ‚(:Ý  ȦÅӈ˃) disk./.usagePercent‚32.79185188373559 disk.drbd2111.in‚0 cpu‚13.86849107965807 disk.drbd1931.out‚0 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0$ proc.memory.rssMB‚11254.03515625* proc.memory.percent‚17.172294855117798 network.recv‚260827$ disk./.usageGB‚586.3516159057617 disk.drbd1965.out‚0 disk.drbd2041.out‚0 disk.drbd1550.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd2118.in‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.drbd1550.out‚0 disk.drbd1356.out‚0 disk.sda1.in‚0 disk.sda2.in‚ 0.41015625 disk.drbd1285.in‚0 disk.drbd1673.out‚0 disk.sda2.out‚ 49.48046875* proc.memory.availableMB‚38515.57421875 network.sent‚ 3.343987e+06 disk.drbd1552.out‚0 disk.drbd1289.in‚0 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.drbd1226.in‚0 proc.cpu.threads‚29 disk.sda.in‚ 0.41015625 disk.dm-0.in‚2.46875 disk.drbd1938.in‚0$ disk.nvme0n1.out‚587.16455078125 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd1938.out‚0 disk.drbd2078.in‚0 disk.rbd1.out‚ 6.1748046875 disk.drbd1285.out‚0 disk.drbd1226.out‚0! disk.dm-0.out‚587.16455078125 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.sda.out‚ 49.48046875 disk.drbd1817.out‚0 disk.nvme0n1.in‚2.46875$ memory_percent‚41.22989773750305\„ I2¹ - _wandbruntime‚1532  _runtime‚1532Ó¬=íǺ Á & lr-AdamW/pg2‚9.275409916061665e-05 trainer/global_step‚3599 % _timestamp‚1.7858117875820777e+09 _runtime‚1532.772018787 _step‚284 & lr-AdamW/pg1‚9.275409916061665e-055ű˜Ì» Æ & lr-AdamW/pg1‚9.275409916061665e-05 & lr-AdamW/pg2‚9.275409916061665e-05 trainer/global_step‚3599 % _timestamp‚1.7858117875820777e+09 _runtime‚1532.772018787 _step‚284œÀ§–2¼ - _wandbruntime‚1532  _runtime‚1532¬Xù¾'½ ¸O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61932170 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10311299 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3159566248996 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38022449 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚243 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 trainer/global_step‚3599 _runtime‚1532.776626322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23919379 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3167405532160 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14175597 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚63428834120952 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3167318102528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚63428731278468 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14175595 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23919377 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚61804872469504 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60269165029472 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚64972190572032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38012793 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚61804889508864 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24486894 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚65204433414144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3159652052120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚64972190572032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚285 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚61804872469504 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10311540 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3253121370624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚64972295041024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38012793 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚61951469718528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚61804889508864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24487137 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61941828 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚61951444814848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 % _timestamp‚1.7858117875830293e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23919377 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚60269182068832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3252988599296 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚64972190572032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚285 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚61804872469504 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10311540 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3253121370624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚64972295041024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38012793 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚61951469718528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚61804889508864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24487137 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61941828 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚61951444814848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 % _timestamp‚1.7858117875830293e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23919377 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚241 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚60269182068832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3252988599296 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10311299 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3159566248996 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38022449 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚243 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 trainer/global_step‚3599 _runtime‚1532.776626322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23919379 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3167405532160 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14175597 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚63428834120952 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3167318102528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚63428731278468 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14175595 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23919377 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚61804872469504 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60269165029472da^Î2¿ - _wandbruntime‚1533  _runtime‚1533ùðXXÿ À ÂM _runtime‚1533.133228496 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61959008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38023316 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚61968676835328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.F¥Épeak‚86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3254038976512 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚61822056271872 trainer/global_step‚3599 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23926022 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3168285158912 % _timestamp‚1.7858117879429474e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3168285158912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61949336 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24493963 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚60285922241152 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61968651931648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚63446451766606 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23926022 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚61822039232512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 _step‚286 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚65222556046848 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3253904115200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38032986 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3168197722112 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23926020 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38032986 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚63446348924010 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3168197722112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚65222715811840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24493718 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10314186 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3160443722218 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61959008 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61949336 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2ø<)¤Í&Á ÇM QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61949336 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24493963 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚60285922241152 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61968651931648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚63446451766606 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23926022 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚61822039232512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 _step‚286 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚65222556046848 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3253904115200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38032986 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3168197722112 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23926020 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38032986 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚63446348924010 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3168197722112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚65222715811840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24493718 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10314186 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3160443722218 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61959008 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61949336 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _runtime‚1533.133228496 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61959008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38023316 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚61968676835328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3254038976512 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚61822056271872 trainer/global_step‚3599 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23926022 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3168285158912 % _timestamp‚1.7858117879429474e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3168285158912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0žŒÏÍ2 - _wandbruntime‚1533  _runtime‚1533ð7à ÿ 2loss/SRqqt1/assignment_loss‚1.4979638768027919 0loss/FBt1/assignment_loss‚0.6135855295869987 _runtime‚1533.133962185 2loss/SRqqt2/assignment_loss‚1.3425108318527539 /loss/FBt2/detection_loss‚1.8264071023756219 0loss/FRt1/assignment_loss‚3.9933668793858708 0loss/FRt2/assignment_loss‚3.6231502202840953 .loss/FRt2/detection_loss‚2.682607818872501 1loss/SRqqt2/detection_loss‚2.3783890315980623 &loss/total_loss‚23.971625091455763 trainer/global_step‚3599 % _timestamp‚1.7858117879441872e+09 2loss/FBt2/assignment_loss‚0.039286697454927993 /loss/FRt1/detection_loss‚2.6352927233721757 0loss/SRqqt1/detection_loss‚1.019858440100219 .loss/FBt1/detection_loss‚2.319205939769745 epoch‚0 _step‚287Ok³ˆ Ä „ 0loss/SRqqt1/detection_loss‚1.019858440100219 .loss/FBt1/detection_loss‚2.319205939769745 epoch‚0 _step‚287 2loss/SRqqt1/assignment_loss‚1.4979638768027919 0loss/FBt1/assignment_loss‚0.6135855295869987 _runtime‚1533.133962185 2loss/SRqqt2/assignment_loss‚1.3425108318527539 /loss/FBt2/detection_loss‚1.8264071023756219 0loss/FRt1/assignment_loss‚3.9933668793858708 0loss/FRt2/assignment_loss‚3.6231502202840953 .loss/FRt2/detection_loss‚2.682607818872501 1loss/SRqqt2/detection_loss‚2.3783890315980623 &loss/total_loss‚23.971625091455763 trainer/global_step‚3599 % _timestamp‚1.7858117879441872e+09 2loss/FBt2/assignment_loss‚0.039286697454927993 /loss/FRt1/detection_loss‚2.6352927233721757ŸÞ‹äp‘Å ‚(:† צÅÓšÔ†ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚130.389' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.463 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¹(¿ÌeÆ ‚(:Ú  צÅÓ¸¼Íƒ disk.drbd1938.in‚0 disk.drbd1226.out‚0 disk.rbd0.out‚0 disk.drbd1552.out‚0 disk.sda2.in‚ 0.41015625 disk.rbd0.in‚0 disk.drbd1285.in‚0! disk.dm-0.out‚587.36767578125* disk./.usagePercent‚32.791856790505925$ disk./.usageGB‚586.3517036437988 disk.drbd1584.out‚0 disk.sda.out‚ 49.953125 disk.dm-0.in‚2.46875 disk.rbd1.in‚0 disk.drbd1965.in‚0$ memory_percent‚41.23035669326782* proc.memory.percent‚17.172294855117798 disk.rbd1.out‚ 6.2685546875 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.drbd1817.in‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1828.in‚0 disk.sda1.out‚0 network.sent‚ 3.347025e+06 disk.drbd1938.out‚0 disk.sda.in‚ 0.41015625 disk.drbd1828.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1226.in‚0 disk.drbd2078.in‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd2111.out‚0 disk.sda1.in‚0) proc.memory.availableMB‚ 38515.2734375 network.recv‚261695 disk.nvme0n1.in‚2.46875 disk.drbd1965.out‚0 disk.sda2.out‚ 49.953125 disk.drbd2041.out‚0 disk.drbd1550.in‚0 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd1510.in‚0 proc.cpu.threads‚29 disk.drbd1356.out‚0 disk.drbd1931.in‚0$ disk.nvme0n1.out‚587.36767578125 disk.drbd1285.out‚0 cpu‚13.866606942585152Ë·²2Ç - _wandbruntime‚1553  _runtime‚1553ÒHý{ÅÈ ¿ & lr-AdamW/pg1‚9.404270848488195e-05 & lr-AdamW/pg2‚9.404270848488195e-05 trainer/global_step‚3649 $ _timestamp‚1.785811808327133e+09 _runtime‚ 1553.51707151 _step‚288"ÄñÊÉ Ä _runtime‚ 1553.51707151 _step‚288 & lr-AdamW/pg1‚9.404270848488195e-05 & lr-AdamW/pg2‚9.404270848488195e-05 trainer/global_step‚3649 $ _timestamp‚1.785811808327133e+09 ICOÌ2Ê - _wandbruntime‚1553  _runtime‚1553D¹p‚¾'Ë ¸O XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚62663210619904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚66109953306112 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚62663210619904 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10454452 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38550451 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3298152651264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3211314237952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24251527 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3211314237952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14372447 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3298289616896 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚62663227659264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3203540515628 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚65874524857856 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14372445 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3203454712504 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚289 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 % _timestamp‚1.7858118083280525e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚62663227659264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3211401667584 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚65874524857856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24251529 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚66110115175424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0’ø¨­'Ì ½O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38540795 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚64309628357976 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚62792322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38550451 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24826640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10454195 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚6281182WÄâœ!5558528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _runtime‚1553.521643742 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚62811800654848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚64309731200460 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚65874629326848 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24826899 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚61106190684832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚65874629326848 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚65874524857856 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14372445 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3203454712504 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚289 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 % _timestamp‚1.7858118083280525e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚62663227659264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3211401667584 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚65874524857856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24251529 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚66110115175424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚62663210619904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚66109953306112 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚62663210619904 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10454452 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38550451 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3298152651264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3211314237952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24251527 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3211314237952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14372447 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3298289616896 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚62663227659264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3203540515628 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3212193857536 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38551318 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3212193857536 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚62680377382912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62819160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38560988 trainer/global_step‚3649 _runtime‚1553.874694147 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24258172 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24258172 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚62829007771648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 % _timestamp‚1.7858118086843636e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚64327348846114 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3212281294336 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24833665 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3204417988962 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3204332185726 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚65892675716608 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚65892571240448 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚64327246003518 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3299188184576 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚62680394422272 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚62680377382912 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62819160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10457012 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62809488 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚65892675716608 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24258170 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24833394 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚62829032675328Є+Í&Ï ÇM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62819160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10457012 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62809488 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚65892675716608 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24258170 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24833394 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚62829032675328 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24258170 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3212193857536 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38551318 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3212193857536 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚62680377382912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62819160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38560988 trainer/global_step‚3649 _runtime‚1553.874694147 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24258172 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24258172 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚62829007771648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 % _timestamp‚1.7858118086843636e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚64327348846114 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3212281294336 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24833665 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3204417988962 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3204332185726 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚65892675716608 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚65892571240448 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚64327246003518 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3299188184576 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚62680394422272 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚62680377382912 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392¢MªÖ02Ð - _wandbruntime‚1553  _runtime‚1553®?kÑ û .loss/FRt1/detection_loss‚2.410972077355665 1loss/SRqqt1/detection_loss‚1.0368909031437197 epoch‚0 1loss/SRqqt2/detection_loss‚2.1077090157912326 /loss/FBt1/detection_loss‚2.8748770356178284 _step‚291 1loss/FBt1/assignment_loss‚0.24926238390617073 .loss/FRt2/detection_loss‚2.823992758581083 trainer/global_step‚3649 % _timestamp‚1.7858118086856802e+09 /loss/FRt1/assignment_loss‚3.868021521848791 2loss/FBt2/assignment_loss‚0.031185244940388657 .loss/FBt2/detection_loss‚1.877106584303783 $loss/total_loss‚24.0429308649274 _runtime‚1553.875495038 .loss/FRt2/assignment_loss‚3.83391728793105 2loss/SRqqt1/assignment_loss‚1.4326360334224741 2loss/SRqqt2/assignment_loss‚1.4963600180852108d«Ò € $loss/total_loss‚24.0429308649274 _runtime‚1553.875495038 .loss/FRt2/assignment_loss‚3.83391728793105 2loss/SRqqt1/assignment_loss‚1.4326360334224741 2loss/SRqqt2/assignment_loss‚1.4963600180852108 .loss/FRt1/detection_loss‚2.410972077355665 1loss/SRqqt1/detection_loss‚1.0368909031437197 epoch‚0 1loss/SRqqt2/detection_loss‚2.1077090157912326 /loss/FBt1/detection_loss‚2.8748770356178284 _step‚291 1loss/FBt1/assignment_loss‚0.24926238390617073 .loss/FRt2/detection_loss‚2.823992758581083 trainer/global_step‚3649 % _timestamp‚1.7858118086856802e+09 /loss/FRt1/assignment_loss‚3.868021521848791 2loss/FBt2/assignment_loss‚0.031185244940388657 .loss/FBt2/detection_loss‚1.877106584303783£¤¯›Ó ‚(: æ¦ÅÓ¾ƒ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚122.743' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.91433333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Þê bÔ ‚(:×  æ¦ÅÓ¯ÖÓƒ disk.sda2.in‚ 0.41015625 disk.drbd1584.out‚0 disk.drbd2041.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.drbd1817.in‚0 network.recv‚264286$ disk./.usageGB‚586.3517761230469 disk.drbd1356.in‚0 disk.rbd0.in‚0 disk.sda.out‚ 50.5703125 disk.drbd1550.out‚0 disk.dm-0.out‚ 588.619140625 disk.drbd1938.out‚0 disk.sda2.out‚ 50.5703125 cpu‚13.864989344059675 network.sent‚ 3.392277e+06 disk.drbd2041.in‚0 disk.sda.in‚ 0.41015625 disk.drbd1965.in‚0 disk.sda1.in‚0 disk.drbd1552.out‚0 proc.cpu.threads‚29 disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd1550.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.rbd1.out‚ 6.3310546875 disk.drbd1285.in‚0 disk.drbd1510.in‚0 disk.drbd1817.out‚0* proc.memory.availableMB‚38515.19140625$ memory_percent‚41.23048186302185 disk.drbd1510.out‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1965.out‚0 disk.drbd1289.out‚0" disk.nvme0n1.out‚ 588.619140625 disk.nvme0n1.in‚2.46875( disk./.usagePercent‚32.7918608439249 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.drbd2111.in‚0 disk.drbd1673.out‚0 disk.drbd1356.out‚0 disk.drbd1226.in‚0 disk.drbd2111.out‚0 disk.drbd2118.in‚0 disk.drbd1584.in‚0 disk.drbd2078.out‚0 disk.drbd1828.in‚0 disk.drbd2118.out‚0c¯F2Õ - _wandbruntime‚1574  _runtime‚1574”sÕwÅÖ ¿ trainer/global_step‚3699 $ _timestamp‚1.785811829142878e+09 _runtime‚ 1574.33461115 _step‚292 & lr-AdamW/pg1‚9.533131780914726e-05 & lr-AdamW/pg2‚9.533131780914726e-05¡²ß!Ê× Ä $ _timestamp‚1.785811829142878e+09 _runtime‚ 1574.33461115 _step‚292 & lr-AdamW/pg1‚9.533131780914726e-05 & lr-AdamW/pg2‚9.533131780914726e-05 trainer/global_step‚3699¤œ—ã82Ø - _wandbruntime‚1574  _runtime‚1574"¯G$Ù ¸O % _timestaqó è¬'mp‚1.7858118291435502e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚66776859143680 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚63521565809664 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3343381678592 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _runtime‚1574.334689253 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚63672156494848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39078453 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚63521548770304 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39078453 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚3699 _step‚293 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24583679 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚66776963612672 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚63521565809664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚63652474 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚63672181398528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚65190525437484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3247428979136 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3247343176012 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3255310373376 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39068797 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24583679 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14569297 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14569295 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3343516547072 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360$b•GÃ'Ú ½O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24583679 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14569297 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14569295 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3343516547072 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7858118291435502e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚66776859143680 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚63521565809664 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3343381678592 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _runtime‚1574.334689253 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚63672156494848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39078453 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚63521548770304 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39078453 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚3699 _step‚293 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24583679 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚66776963612672 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚63521565809664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚63652474 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚63672181398528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚65190525437484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3247428979136 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3247343176012 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3255310373376 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39068797 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130¥èàzr2Û - _wandbruntime‚1574  _runtime‚1574­|*gÈ&Ü ÂM =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚63538715533312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚3699 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3248306452470 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39079320 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3256189992960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39088990 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14573234 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚65208245925622 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14573232 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3256277429760 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3256189992960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚230 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24590322 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚63538732572672 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚63538715533312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25174302 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚65208143083026 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚1574.681345313 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3344287364096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24590320 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3256277429760 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚63689363611648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚61959922433792 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63679312 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24590322 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚67033806546432 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚61959939473152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25174072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0ÉBä6t Ý ÇM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚63538715533312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25174302 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚65208143083026 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚1574.681345313 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3344287364096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24590320 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3256277429760 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚63689363611648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚61959922433792 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.s"ênYmall_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63679312 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24590322 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚67033806546432 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚61959939473152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25174072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚63538715533312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚3699 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3248306452470 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39079320 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3256189992960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39088990 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14573234 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚65208245925622 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14573232 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3256277429760 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3256189992960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚230 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24590322 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚63538732572672 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640¦tyÑ­2Þ - _wandbruntime‚1574  _runtime‚1574=‚ÒŽß ÿ _step‚295 2loss/SRqqt1/assignment_loss‚1.5034018905058395 2loss/SRqqt2/assignment_loss‚1.7148448502364224 0loss/FBt2/assignment_loss‚0.2370165884888801 /loss/FRt1/detection_loss‚2.5650591524442037 /loss/FBt2/detection_loss‚1.8758777963057056 epoch‚0 % _timestamp‚1.7858118294922829e+09 /loss/FRt2/assignment_loss‚3.647215256648781 1loss/SRqqt1/detection_loss‚0.8826526600136214 0loss/SRqqt2/detection_loss‚2.172610023454444 /loss/FBt1/detection_loss‚4.1953255619321554 _runtime‚1574.682053683 0loss/FRt1/assignment_loss‚3.9381929000218707 1loss/FBt1/assignment_loss‚0.20748104519381222 /loss/FRt2/detection_loss‚2.2184017730497683 &loss/total_loss‚25.158079498295503 trainer/global_step‚3699/>S± à „ _step‚295 2loss/SRqqt1/assignment_loss‚1.5034018905058395 2loss/SRqqt2/assignment_loss‚1.7148448502364224 0loss/FBt2/assignment_loss‚0.2370165884888801 /loss/FRt1/detection_loss‚2.5650591524442037 /loss/FBt2/detection_loss‚1.8758777963057056 epoch‚0 % _timestamp‚1.7858118294922829e+09 /loss/FRt2/assignment_loss‚3.647215256648781 1loss/SRqqt1/detection_loss‚0.8826526600136214 0loss/SRqqt2/detection_loss‚2.172610023454444 /loss/FBt1/detection_loss‚4.1953255619321554 _runtime‚1574.682053683 0loss/FRt1/assignment_loss‚3.9381929000218707 1loss/FBt1/assignment_loss‚0.20748104519381222 /loss/FRt2/detection_loss‚2.2184017730497683 &loss/total_loss‚25.158079498295503 trainer/global_step‚3699§€wÿ›á ‚(: õ¦ÅÓͳƒ gpu.0.gpu‚30.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚118.93' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚39.64333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0`9øeâ ‚(:Ú  õ¦ÅÓú‡ìƒ! disk.dm-0.out‚594.12060546875 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.drbd1285.in‚0 disk.drbd1931.out‚0 cpu‚13.86339894568663 disk.drbd1673.in‚0 proc.cpu.threads‚29 disk.sda.in‚ 0.4140625 disk.drbd1965.out‚0 disk.sda1.in‚0 disk.drbd1817.out‚0 network.recv‚266697 disk.drbd1931.in‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.dm-0.in‚2.46875# memory_percent‚41.2308394908905) disk./.usagePercent‚32.79183716342457 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd1938.out‚0 disk.rbd0.out‚0 network.sent‚ 3.437223e+06 disk.drbd1356.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1226.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.sda2.out‚ 54.44140625 disk.sda.out‚ 54.44140625 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0$ disk./.usageGB‚586.3513526916504$ disk.nvme0n1.out‚594.12060546875 disk.drbd1584.in‚0 disk.rbd1.in‚0 disk.drbd1226.out‚0 disk.rbd1.out‚ 6.4248046875 disk.sda2.in‚ 0.4140625 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0* proc.memory.availableMB‚38514.95703125$ proc.memory.rssMB‚11254.03515625 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd1356.in‚0ywo‘ã ‚(:† „§ÅÓ «ƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚102.231' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚34.077 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0WÑ`ä ‚(:Õ  „§ÅÓª æƒ disk.sda2.in‚ 0.4140625' proc.memory.availableMB‚ 38515.09375 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0 disk.drbd1552.out‚0 disk.drbd1226.out‚0 disk.rbd0.out‚0 disk.dm-0.out‚ 596.919921875 disk.drbd1289.out‚0 disk.drbd1817.out‚0* proc.memory.percent‚17.172294855117798$ memory_percent‚41.23063087463379 disk.drbd1285.in‚0 disk.drbd2118.out‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.4140625 disk.drbd1226.in‚0 disk.sda1.in‚0 disk.sda.out‚ 56.03515625 disk.drbd1550.in‚0 disk.sda1.out‚0 network.recv‚269115 disk.rbd0.in‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 56.03515625 cpu‚13.861387556933646 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd1965.out‚0 disk.drbd2078.out‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.rbd1.out‚ 6.4873046875" disk.nvme0n1.out‚ 596.919921875 disk.drbd1584.out‚0 disk.drbd2078.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1550.out‚0) disk./.usagePercent‚32.79181668299185 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd2041.out‚0$ disk./.usageGB‚586.3509864807129 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd2111.out‚0 disk.dm-0.in‚2.46875 disk.drbd1289.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd2041.in‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1817.in‚0 disk.drbd1673.out‚0 proc.cpu.threads‚29 network.sent‚ 3.482227e+06 disk.drbd1828.in‚0 disk.drbd1552.in‚06+2k2å - _wandbruntime‚1595  _runtime‚1595»õ»ëÆæ À _runtime‚1595.494785133 _step‚296 & lr-AdamW/pg1‚9.661992713341257e-05 & lr-AdamW/pg2‚9.661992713341257e-05 trainer/global_step‚3749 $ _timestamp‚1.785811850304845e+09xVMîËç Å & lr-AdamW/pg1‚9.661992713341257e-05 & lr-AdamW/pg2‚9.661992713341257e-05 trainer/global_step‚3749 $ _timestamp‚1.785811850304845e+09 _runtime‚1595.494785133 _step‚296¨´ß~2è - _wandbruntime‚1595  _runtime‚1595ìaÞƒ¾'é ¸O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64522284 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24915829 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14766145 % _timestamp‚1.7858118503057678e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3299393938432 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚64379886920704 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3299393938432 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39606455 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚67679297898496 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3291317442644 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚64379903960064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚3749 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚66071422516992 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24915829 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3299306508800 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚67679297898496 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚62780190877472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3299306508800 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚64379903960064 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39596799 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚64532512334848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚297 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25507313 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10741168 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚67679193429504 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚67921098677760 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚67679193429504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚62780207916832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39596799 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚3749 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚66071422516992 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24915829 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3299306508800 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚67679297898496 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚62780190877472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3299306508800 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚64379903960064 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39596799 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚64532512334848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚297 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25507313 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10741168 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚67679193429504 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚67921098677760 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚67679193429504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚62780207916832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39596799 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39606455 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚67679297898496 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YKDeviceStatsMonitor.on_train_batch_start/reserved_£á ˜cbytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3291317442644 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚64379903960064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640©À¨ç_2ë - _wandbruntime‚1595  _runtime‚1595Ô£©Ç&ì ÁM QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _runtime‚ 1595.84767521 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14770084 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3292109112742 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24922472 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25514211 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 _step‚298 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24922470 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚238 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚64549744355328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚64397053683712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64539464 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3300273565184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64539464 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3749 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚67697239812096 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39616992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3389624655360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚64397053683712 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚67697344288256 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚62796931049792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 % _timestamp‚1.7858118506573694e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24922472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25513973 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚66089040162534 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10743891 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚67939211342848 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚64397070723072 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3300186128384 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39616992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚62796948089152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14770082 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712ÉŒ°àÌ&í ÆM O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚64529792 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚64549719451648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3389491891200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚64397053683712 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚67697344288256 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚62796931049792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 % _timestamp‚1.7858118506573694e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24922472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25513973 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚66089040162534 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10743891 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚67939211342848 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚64397070723072 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3300186128384 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39616992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚62796948089152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14770082 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _runtime‚ 1595.84767521 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14770084 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3292109112742 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24922472 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25514211 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 _step‚298 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24922470 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚238 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚64549744355328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚64397053683712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64539464 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3300273565184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64539464 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3749 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚67697239812096 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39616992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3389624655360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚68581632184320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3433941194240 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3433802131456 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14962997 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚63617216532832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚63617199493472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3343302644224 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40124801 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858118708055954e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10883434 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚65238225071104 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3335205906152 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚68581632184320 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40134457 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 Â(”y| TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚65238242110464 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25247979 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚65238225071104 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚68826670306304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚301 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚65382436 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14962995 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚68581527715328 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚66952422438984 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25846431 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚68826834272768 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032]É°¿Ã'ø ½O SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚68826670306304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚301 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚65382436 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14962995 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚68581527715328 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚66952422438984 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25846431 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚68826834272768 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40124801 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚65392893078528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚68581527715328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40134457 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3343390073856 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3343302644224 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚68581632184320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3433941194240 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3433802131456 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14962997 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚63617216532832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚63617199493472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3343302644224 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40124801 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858118708055954e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10883434 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚65238225071104 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3335205906152 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚68581632184320 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40134457 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚65238242110464 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25247979 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚65238225071104­Í ,â2ù - _wandbruntime‚1616  _runtime‚1616O AmÇ&ú ÁM ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10886215 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚65255391834112 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14966932 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3434708046336 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚63633939665792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3336083379486 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25852915 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65399616 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40144994 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚65255391834112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚68599678574080 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚68844936811520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚65255408873472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚1616.352867695 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚66970040084638 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚65410075291648 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3799 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚65255408873472 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚68599678574080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚232 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚66969937242042 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3344182263808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25254620 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14966934 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40135324 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3344269700608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65389944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65389944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40135324 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚63633956705152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25254620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚234 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚65410100195328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚65399616 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40144994 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚68599574097920 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3335997576250 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3344269700608 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3344182263808 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2‘”¥%û ÆM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25254622 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25254620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚234 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚65410100195328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚65399616 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40144994 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚68599574097920 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3335997576250 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3344269700608 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3344182263808 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10886215 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚65255391834112 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14966932 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3434708046336 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚63633939665792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3336083379486 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25852915 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65399616 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40144994 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚65255391834112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚68599678574080 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚68844936811520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚65255408873472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚1616.352867695 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚66970040084638 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚65410075291648 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3799 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚65255408873472 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚68599678574080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚232 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚66969937242042 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3344182263808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25254620 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14966934 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40135324 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3344269700608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65389944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65389944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40135324 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚63633956705152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40652803 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3387298779648 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15159845 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3479016639488 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚67833216676008 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚69483966470144 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚69483862001152 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40662459 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚66096563221504 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3379008566536 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66242588 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3387298779648 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25580129 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚64454225148832 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25580127 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚66096580260864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3479147313664 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚64454208109472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 trainer/global_step‚3849 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25580127 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11026187 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _step‚305 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40662459 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚67833319518492 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11026430 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3379094369660 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚69483862001152 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26186032 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚66253224014848 % _timestamp‚1.7858118918984594e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚66096563221504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26186277 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2N0!ßÃ'ˆ ½O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11026187 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _step‚305 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40662459 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚67833319518492 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11026430 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3379094369660 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚69483862001152 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26186032 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚66253224014848 % _timestamp‚1.7858118918984594e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚66096563221504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26186277 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚69732240654336 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15159847 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25580129 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40652803 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3387298779648 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15159845 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3479016639488 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚67833216676008 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚69483966470144 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚69483862001152 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40662459 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚66096563221504 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3379008566536 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66242588 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3387298779648 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25580129 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚64454225148832 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25580127 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚66096580260864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3479147313664 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚64454208109472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 trainer/global_step‚3849 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25580127 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712±Ï=Òf2‰ - _wandbruntime‚1637  _runtime‚1637yÚùʘŠ ÁM I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚69502012859904 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚66113747023872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚66270456035328 trainer/global_step‚3849 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11029062 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚66270431131648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3480055662592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚67850834321550 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚66250096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3388265836032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚69750511697920 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚69502012859904 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25586770 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3388265836032 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚306 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15163784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚69501908383744 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40672996 _runtime‚1637.435746841 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25586772 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3479912412672 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3379971842994 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11029322 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3388178399232 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40663326 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26192844 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚64470948281792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66259768 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚66113729984512 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚66250096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚64470965321152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SCŽ½¿/EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66259768 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚67850937164146 $ _timestamp‚1.785811892245458e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚66113747023872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25586772 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40663326 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚66113729984512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3379886039758 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚69750343544320«›zÌ&‹ ÆM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚67850834321550 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚66250096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3388265836032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚69750511697920 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚69502012859904 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25586770 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3388265836032 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚306 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15163784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚69501908383744 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40672996 _runtime‚1637.435746841 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25586772 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3479912412672 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3379971842994 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11029322 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3388178399232 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40663326 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26192844 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚64470948281792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66259768 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚66113729984512 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚66250096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚64470965321152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66259768 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚67850937164146 $ _timestamp‚1.785811892245458e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚66113747023872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25586772 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40663326 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚66113729984512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3379886039758 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚69750343544320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚69502012859904 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚66113747023872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚66270456035328 trainer/global_step‚3849 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11029062 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚66270431131648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3480055662592²S¤y¹2Œ - _wandbruntime‚1637  _runtime‚1637o/ ü 0loss/FRt1/assignment_loss‚4.3626490227783785 1loss/SRqqt1/detection_loss‚0.9849896092131178 trainer/global_step‚3849 _runtime‚1637.436632251 _step‚307 0loss/FBt1/assignment_loss‚0.4851650083437562 /loss/FRt1/detection_loss‚2.5143908198121228 /loss/SRqqt2/detection_loss‚2.27056593199571 % _timestamp‚1.7858118922466664e+09 /loss/FRt2/assignment_loss‚3.508294888571197 1loss/FBt2/assignment_loss‚0.11733679988293359 /loss/FRt2/detection_loss‚2.2839667116310083 /loss/FBt2/detection_loss‚1.1556418578893521 0loss/SRqqt1/assignment_loss‚1.78576696294922 2loss/SRqqt2/assignment_loss‚1.5987842960490122 /loss/FBt1/detection_loss‚2.6669681626077537 &loss/total_loss‚23.734520071723562 epoch‚0;¦,¼Ž  0loss/FBt1/assignment_loss‚0.4851650083437562 /loss/FRt1/detection_loss‚2.5143908198121228 /loss/SRqqt2/detection_loss‚2.27056593199571 % _timestamp‚1.7858118922466664e+09 /loss/FRt2/assignment_loss‚3.508294888571197 1loss/FBt2/assignment_loss‚0.11733679988293359 /loss/FRt2/detection_loss‚2.2839667116310083 /loss/FBt2/detection_loss‚1.1556418578893521 0loss/SRqqt1/assignment_loss‚1.78576696294922 2loss/SRqqt2/assignment_loss‚1.5987842960490122 /loss/FBt1/detection_loss‚2.6669681626077537 &loss/total_loss‚23.734520071723562 epoch‚0 0loss/FRt1/assignment_loss‚4.3626490227783785 1loss/SRqqt1/detection_loss‚0.9849896092131178 trainer/global_step‚3849 _runtime‚1637.436632251 _step‚307³N\{‚ ‚(:’ À§ÅӞݦƒ gpu.0.gpu‚43.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚119.071' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚39.690333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0{9ü‘a ‚(:Ö  À§ÅÓÂ…çƒ disk.sda.out‚ 57.4609375 disk.drbd1226.in‚0$ disk./.usageGB‚586.3500671386719 disk.drbd1356.out‚0 disk.drbd2111.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd2078.out‚0 disk.drbd1673.in‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.sda.in‚ 0.4140625 disk.drbd1965.in‚0 network.sent‚ 3.578199e+06 disk.drbd1510.out‚0 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1356.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.drbd1289.in‚0* proc.memory.availableMB‚38516.34765625 cpu‚13.853618762703602 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.rbd1.out‚ 6.67578125 disk.dm-0.out‚598.6396484375 disk.drbd1552.in‚0 network.recv‚276006 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.rbd0.out‚0 disk.drbd1584.out‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 57.4609375# disk.nvme0n1.out‚598.6396484375 disk.drbd2041.in‚0 disk.drbd1285.in‚0 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.drbd1965.out‚0 disk.drbd1285.out‚0 disk.drbd1510.in‚0 disk.drbd1938.out‚0# memory_percent‚41.2287175655365 proc.cpu.threads‚29* disk./.usagePercent‚32.791765268572206 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.sda2.in‚ 0.4140625 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.drbd1673.out‚0* proc.memory.percent‚17.172294855117798ì.íÿ2‘ - _wandbruntime‚1657  _runtime‚1657%^ mÉ’ à ' lr-AdamW/pg1‚0.00010048575510620847 ' lr-AdamW/pg2‚0.00010048575510620847 trainer/global_step‚3899 % _timestamp‚1.7858119126440017e+09 _runtime‚1657.835621911 _step‚308_*½ Γ È trainer/global_step‚3899 % _timestamp‚1.7858119126440017e+09 _runtime‚1657.835621911 _step‚308 ' lr-AdamW/pg1‚0.00010048575510620847 ' lr-AdamW/pg2‚0.00010048575510620847´p·F 2” - _wandbruntime‚1657  _runtime‚1657ølQ¾'• ¸O _runtime‚1657.835700381 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11168965 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 % _timestamp‚1.7858119126446714e+09 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚66954901371904 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67102740 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3524306199040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26525660 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚70637910957568 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3431294915072 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26525930 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚70386196286976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67113604758528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚3899 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15356695 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11169233 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚70386300755968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41190461 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3431294915072 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67093082 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚70386300755968 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚66954901371904 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15356697 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41180805 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚268 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25912279 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25912277 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3524171330560 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚70386196286976 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41190461 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚65291216725472 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25912277 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _step‚309 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚65291216725472 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25912277 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _step‚309 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚70386196286976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67113604758528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚3899 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15356695 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11169233 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚70386300755968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41190461 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3431294915072 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67093082 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚70386300755968 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚66954901371904 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15356697 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41180805 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚268 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25912279 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25912277 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3524171330560 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚70386196286976 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41190461 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3432261971456 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚68731834243654 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚70404347145728 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67119920 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚68731731401058 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3525207388160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25918920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3525076721152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚66972085174272 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚70404242669568 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚67130786971648 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚66972085174272 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚66972068134912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3423860306502 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 trainer/global_step‚3899 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3432174534656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26532394 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _runtime‚1658.184341358 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3432261971456 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11171762 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25918922 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15360634 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3423774503266 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚65307973937152 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41200998 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26532641 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3432174534656 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚310 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41191328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15360632 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67110248 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41191328 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3432261971456 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚68731834243654 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚70404347145728 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67119920 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚68731731401058 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3525207388160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25918920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3525076721152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚66972085174272 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚70404242669568 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚67130786971648 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚66972085174272 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚66972068134912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3423860306502 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 trainer/global_step‚3899 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3432174534656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26532394 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _runtime‚1658.184341358 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3432261971456 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11171762 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25918922 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15360634 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3423774503266 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚65307973937152 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41200998 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26532641 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3432174534656 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚310 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41191328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15360632 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67110248 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41191328 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251¶6Y³ø2š - _wandbruntime‚1658  _runtime‚1658[ÊUõ› ÿ 1loss/SRqqt2/assignment_loss‚1.596329838998856 1loss/FBt1/assignment_loss‚0.37493256404801895 1loss/FBt2/assignment_loss‚0.14387278594787176 /loss/FRt2/detection_loss‚2.5235735072243597 $ _timestamp‚1.785811912995282e+09 /loss/FBt2/detection_loss‚1.7741650750860571 epoch‚0 trainer/global_step‚3899 0loss/SRqqt2/detection_loss‚2.307295445953646 /loss/FBt1/detection_loss‚3.5373124281565347 &loss/total_loss‚24.394245517023293 0loss/FRt1/assignment_loss‚3.7371285313215012 2loss/SRqqt1/assignment_loss‚1.3879530455727698 /loss/FRt1/detection_loss‚2.3946100866947417 1loss/SRqqt1/detection_loss‚1.0222821749040811 _runtime‚1658.185075341 0loss/FRt2/assignment_loss‚3.5947900331148537 _step‚311>SQ œ „ 1loss/FBt2/assignment_loss‚0.14387278594787176 /loss/FRt2/detection_loss‚2.5235735072243597 $ _timestamp‚1.785811912995282e+09 /loss/FBt2/detection_loss‚1.7741650750860571 epoch‚0 trainer/global_step‚3899 0loss/SRqqt2/detection_loss‚2.307295445953646 /loss/FBt1/detection_loss‚3.5373124281565347 &loss/total_loss‚24.394245517023293 0loss/FRt1/assignment_loss‚3.7371285313215012 2loss/SRqqt1/assignment_loss‚1.3879530455727698 /loss/FRt1/detection_loss‚2.3946100866947417 1loss/SRqqt1/detection_loss‚1.0222821749040811 _runtime‚1658.185075341 0loss/FRt2/assignment_loss‚3.5947900331148537 _step‚311 1loss/SRqqt2/assignment_loss‚1.596329838998856 1loss/FBt1/assignment_loss‚0.37493256404801895·³f˜äœ ‚(:‘ ϧÅÓ³Ó’ƒ gpu.0.gpu‚12.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚94.407' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚31.468999999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0}ãJbž ‚(:×  ϧÅÓöç؃ disk.drbd2118.out‚0( disk./.usagePercent‚32.7917631351938 disk.drbd1817.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.sda.in‚ 0.4140625 disk.rbd0.out‚0 disk.dm-0.out‚598.8662109375 disk.sda1.in‚0 disk.sda2.in‚ 0.4140625 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.drbd1510.in‚0* proc.memory.availableMB‚38516.31640625 cpu‚13.851853165334653 disk.drbd1938.out‚0 disk.sda2.out‚ 57.83984375 disk.drbd2078.in‚0 disk.drbd1828.in‚0$ proc.memory.rssMB‚11254.03515625 proc.cpu.threads‚29$ disk./.usageGB‚586.3500289916992 disk.drbd1931.out‚0 disk.drbd1965.out‚0 disk.drbd1828.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd1673.out‚0# disk.nvme0n1.out‚598.8662109375 disk.drbd1584.in‚0 disk.drbd1285.in‚0 disk.drbd1817.out‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.nvme0n1.in‚2.46875 disk.rbd1.out‚ 6.85546875 network.sent‚ 3.623147e+06 disk.drbd2111.in‚0 disk.drbd1289.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.dm-0.in‚2.46875 disk.sda1.out‚0$ memory_percent‚41.22876524925232 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.sda.out‚ 57.83984375 disk.rbd1.in‚0 network.recv‚278331 disk.drbd1584.out‚0 disk.drbd2111.out‚0 disk.drbd1226.out‚0 disk.drbd1552.in‚0U£êñ2Ÿ - _wandbruntime‚1678  _runtime‚1678SFA-É  à _step‚312 ' lr-AdamW/pg1‚0.00010177436443047379 ' lr-AdamW/pg2‚0.00010177436443047379 trainer/global_step‚3949 % _timestamp‚1.7858119334661987e+09 _runtime‚1678.656146431YÄ¢¾Î¡ È ' lr-AdamW/pg1‚0.00010177436443047379 ' lr-AdamW/pg2‚0.00010177436443047379 trainer/global_step‚3949 % _timestamp‚1.7858119334661987e+09 _runtime‚1678.656146431 _step‚312¸5ý2¢ - _wandbruntime‚1678  _runtime‚16782N0¾’ £ ¸O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3475291050496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚67813256561664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26244427 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚313 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26244427 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚71288530572800 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚71543417964032 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚67813239522304 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚661282253èñ²á,41472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858119334671257e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚67813239522304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41718463 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚67813256561664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15553547 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67973960598528 _runtime‚1678.660634168 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚71288635041792 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41708807 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3475378480128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3475291050496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11312190 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚69595113677508 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3466785493552 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3466871296676 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚71288530572800 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚71543258191872 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚3949 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3475378480128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26865986 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚67973935694848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3569457365504 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15553545 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26865735 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚66128242380832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚249 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26244429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41718463 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41708807 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67962892 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚71288635041792­Xè Ã'¤ ½O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15553545 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26865735 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚66128242380832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚249 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26244429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41718463 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41708807 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67962892 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚71288635041792 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3475291050496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚67813256561664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26244427 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚313 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26244427 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚71288530572800 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚71543417964032 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚67813239522304 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚66128225341472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858119334671257e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚67813239522304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41718463 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚67813256561664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15553547 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67973960598528 _runtime‚1678.660634168 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚71288635041792 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41708807 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3475378480128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3475291050496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11312190 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚69595113677508 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3466785493552 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3466871296676 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚71288530572800 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚71543258191872 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚3949 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3475378480128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26865986 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚67973935694848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3569457365504¹:ÿõ2¥ - _wandbruntime‚1679  _runtime‚1679òǘÇ&¦ ÁM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚67830406285312 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26251070 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11315241 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67970400 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚67991142811648 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚71306681431552 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚71306681431552 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚69612628480566 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26251070 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 $ _timestamp‚1.785811933820828e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3476170670080 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67980072 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚71561524000768 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41719330 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚3949 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26251072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3570356285440 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚66144965513792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3476170670080 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚66144982553152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67980072 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41719330 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚67830423324672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚67830406285312 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚69612731323162 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3467748770010 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3476258106880 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26872471 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15557484 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15557482 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚67991167715328 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1679.011115162 _step‚314 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41729000 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3467662966774 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67970400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41729000 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3570231909888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3476258106880 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26872725 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1ž &õ§ ÆM [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚67830406285312 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26251070 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11315241 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67970400 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚67991142811648 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚71306681431552 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚71306681431552 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚69612628480566 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26251070 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 $ _timestamp‚1.785811933820828e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3476170670080 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67980072 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚71561524000768 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41719330 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚3949 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26251072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3570356285440 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚66144965513792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3476170670080 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/r§£—G×equested_bytes.large_pool.allocated‚66144982553152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67980072 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41719330 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚67830423324672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚67830406285312 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚69612731323162 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3467748770010 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3476258106880 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26872471 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15557484 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15557482 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚67991167715328 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1679.011115162 _step‚314 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41729000 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3467662966774 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67970400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41729000 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3570231909888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3476258106880 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26872725 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712º¸ Q‹2¨ - _wandbruntime‚1679  _runtime‚1679°^¿Ü© ý /loss/FBt2/detection_loss‚1.6644527620588807 trainer/global_step‚3949 % _timestamp‚1.7858119338220484e+09 0loss/FRt1/assignment_loss‚3.9202055628619976 0loss/FRt2/assignment_loss‚3.7183604265514174 1loss/SRqqt2/assignment_loss‚1.488902420472743 0loss/FBt1/assignment_loss‚0.5105143918313283 1loss/SRqqt2/detection_loss‚2.1150240078012446 /loss/FBt1/detection_loss‚1.6754858539654658 _step‚315 /loss/FRt1/detection_loss‚2.7434166354029927 0loss/SRqqt1/detection_loss‚1.001420448916467 &loss/total_loss‚22.992471981680517 1loss/SRqqt1/assignment_loss‚1.564957305907257 .loss/FRt2/detection_loss‚2.518141646134226 epoch‚0 _runtime‚1679.011836992 1loss/FBt2/assignment_loss‚0.07159051977649594«f•ª ‚ _step‚315 /loss/FRt1/detection_loss‚2.7434166354029927 0loss/SRqqt1/detection_loss‚1.001420448916467 &loss/total_loss‚22.992471981680517 1loss/SRqqt1/assignment_loss‚1.564957305907257 .loss/FRt2/detection_loss‚2.518141646134226 epoch‚0 _runtime‚1679.011836992 1loss/FBt2/assignment_loss‚0.07159051977649594 /loss/FBt2/detection_loss‚1.6644527620588807 trainer/global_step‚3949 % _timestamp‚1.7858119338220484e+09 0loss/FRt1/assignment_loss‚3.9202055628619976 0loss/FRt2/assignment_loss‚3.7183604265514174 1loss/SRqqt2/assignment_loss‚1.488902420472743 0loss/FBt1/assignment_loss‚0.5105143918313283 1loss/SRqqt2/detection_loss‚2.1150240078012446 /loss/FBt1/detection_loss‚1.6754858539654658»h¯SÒœ« ‚(:‘ Þ§ÅÓ𬠃 gpu.0.gpu‚20.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚120.784' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.26133333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿ûWI`¬ ‚(:Õ  Þ§ÅÓÀÉéƒ disk.drbd1938.in‚0 disk.drbd2111.in‚0 disk.drbd2118.in‚0 disk.drbd1289.in‚0* proc.memory.percent‚17.172294855117798 disk.drbd1938.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd2078.out‚0 disk.dm-0.out‚599.1904296875 disk.sda1.in‚0 disk.drbd1817.in‚0$ disk./.usageGB‚586.3500289916992 disk.drbd1817.out‚0 disk.sda.in‚ 0.4140625 disk.drbd2078.in‚0 disk.drbd1510.out‚0 disk.drbd1552.out‚0 disk.drbd1931.in‚0 disk.drbd1285.in‚0 disk.drbd1289.out‚0 disk.drbd2118.out‚0 disk.drbd1356.out‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd2111.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.drbd1965.in‚0( disk./.usagePercent‚32.7917631351938 disk.rbd1.in‚0$ memory_percent‚41.22918248176575 disk.rbd1.out‚ 6.91796875 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.sda.out‚ 58.1953125 disk.drbd1828.in‚0 disk.drbd1510.in‚0# disk.nvme0n1.out‚599.1904296875 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 proc.cpu.threads‚29 network.recv‚280754 disk.drbd1226.out‚0 disk.dm-0.in‚2.46875 disk.sda2.out‚ 58.1953125 disk.drbd1356.in‚0 network.sent‚ 3.668112e+06 disk.sda2.in‚ 0.4140625 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0* proc.memory.availableMB‚38516.04296875 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 cpu‚13.849827105947563ýx7œ­ ‚(:‘ í§ÅÓ¯•›ƒ gpu.0.gpu‚17.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚94.504' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚31.501333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0º¶± `® ‚(:Õ  í§ÅӺ disk.sda1.out‚0 disk.nvme0n1.in‚2.46875* proc.memory.percent‚17.172294855117798 network.recv‚283132 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1828.in‚0 disk.drbd1289.out‚0$ memory_percent‚41.22849106788635 disk.rbd1.out‚ 6.91796875 disk.drbd1817.out‚0 disk.drbd1938.out‚0 disk.drbd1226.out‚0 disk.drbd1289.in‚0 disk.drbd1673.out‚0 disk.drbd1285.out‚0 network.sent‚ 3.713071e+06( disk./.usagePercent‚32.7917631351938 disk.drbd1552.in‚0 disk.rbd0.out‚0 proc.cpu.threads‚29 disk.drbd1356.out‚0 disk.drbd1510.out‚0 disk.drbd1828.out‚0 disk.drbd1584.in‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.4140625 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.sda1.in‚0 disk.drbd1673.in‚0# disk.nvme0n1.out‚599.3701171875 disk.drbd2041.out‚0 disk.dm-0.out‚599.3701171875 disk.drbd2041.in‚0 disk.drbd2111.in‚0* proc.memory.availableMB‚38516.49609375 disk.sda.in‚ 0.4140625 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.dm-0.in‚2.46875 disk.drbd1510.in‚0 cpu‚13.848270209306493 disk.drbd1552.out‚0 disk.sda2.out‚ 58.5078125 disk.drbd1226.in‚0$ disk./.usageGB‚586.3500289916992 disk.rbd1.in‚0 disk.drbd1550.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1931.out‚0 disk.sda.out‚ 58.5078125 disk.drbd2111.out‚0 disk.drbd1356.in‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1285.in‚0}ëwÜ2¯ - _wandbruntime‚1699  _runtime‚16994/«EÇ° Á & lr-AdamW/pg1‚0.0001030629737547391 & lr-AdamW/pg2‚0.0001030629737547391 trainer/global_step‚3999 % _timestamp‚1.7858119544203427e+09 _runtime‚1699.610269995 _step‚316-+ƒŒÌ± Æ _runtime‚1699.610269995 _step‚316 & lr-AdamW/pg1‚0.0001030629737547391 & lr-AdamW/pg2‚0.0001030629737547391 trainer/global_step‚3999 % _timestamp‚1.7858119544203427e+09¼‚Ä:2² - _wandbruntime‚1699  _runtime‚1699cžèI¾'³ ¸O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42236809 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚70475907914532 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚72190864858624 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚1699.614631157 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26576577 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15750397 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚68671577672704 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15750395 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚72190864858624 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚3999 _step‚317 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚68671594712064 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3519287185920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11455803 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11455547 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3614617868800 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26576579 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68823044 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27206200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚68671577672704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚72448934307328 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚72448772438016 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚66965250996832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3510759760184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7858119544212775e+09 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚68834316438528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3519287185920 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26576579 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3519374615552 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚68671594712064 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚256 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚68834291534848vÛí5Ã'´ ½O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3519374615552 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚68671594712064 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚256 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚68834291534848 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42236809 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚70475907914532 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚72190864858624 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚1699.614631157 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26576577 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15750397 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚68671577672704 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15750395 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚72190864858624 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚3999 _step‚317 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚68671594712064 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3519287185920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11455803 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11455547 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3614617868800 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26576579 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68823044 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27206200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚68671577672704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚72448934307328 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚72448772438016 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚66965250996832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3510759760184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7858119544212775e+09 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚68834316438528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3519287185920 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26576579½yî 2µ - _wandbruntime‚1699  _runtime‚1699Wˆˆ(Ú¶ ÂM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚68688761475072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak.T`€î!‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27212692 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26583222 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15754334 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68840224 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27212946 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚68688761475072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚68851498651648 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚68830552 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42257002 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚72209015717376 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚70493628402670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3511551430282 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26583222 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚72467042416640 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚72466882651648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚70493525560074 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3511637233518 _step‚318 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚68688761475072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27212692 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26583222 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15754334 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68840224 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27212946 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚68688761475072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚68851498651648 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚68830552 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42257002 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚72209015717376 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚70493628402670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3511551430282 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26583222 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚72467042416640 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚72466882651648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚70493525560074 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3511637233518 _step‚318 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚69529915823104 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚73354462231040 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚71356907836524 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26908729 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11597104 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚4049 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15947247 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚69529915823104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26908727 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42774467 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚73354302458880 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚69694647374848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚321 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚71356804994040 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚69529932862464 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27544636 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3563370750976 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚69529932862464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚67802259612832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3659789952512 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42774467 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚69694672278528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 $ _timestamp‚1.785811975185896e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26908727 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚73093199144448 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚73093199144448 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚69529915823104 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚73354462231040 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚71356907836524 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26908729 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11597104 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚4049 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15947247 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚69529915823104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26908727 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42774467 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚73354302458880 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚69694647374848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚321 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚71356804994040 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚69529932862464 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27544636 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3563370750976 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚69529932862464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚67802259612832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3659789952512 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0Á™ü“â2à - _wandbruntime‚1720  _runtime‚1720Ž×À_Æ&Ä ÀM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26915372 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3564250377728 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚69547082586112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15951182 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42775334 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69690704 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27551352 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26915372 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27551100 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 # _timestamp‚1.78581197553576e+09 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚67818982745792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚69690704 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚69547099625472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42775334 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15951184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚69547099625472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚69711879395328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚1720.726038435 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69700376 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚69547082586112 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11599918 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3555439893790 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69700376 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚73372572429312 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚67818999785152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3564162940928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚71374525482178 trainer/global_step‚4049 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11600168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _step‚322 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3555525697026 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26915370 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26915370 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3564162940928 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3564162940928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚71374525482178 trainer/global_step‚4049 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11600168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _step‚322 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3555525697026 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26915370 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26915370 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3564162940928 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26915372 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27551100 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 # _timestamp‚1.78581197553576e+09 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚67818982745792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚69690704 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚69547099625472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42775334 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15951184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚69547099625472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚69711879395328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚1720.726038435 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69700376 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚69547082586112 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024Âe8=2Æ - _wandbruntime‚1720  _runtime‚1720ÔÁô=Ç ú _runtime‚1720.726727694 1loss/SRqqt1/assignment_loss‚1.449181053030297 .loss/FRt2/detection_loss‚2.654911436612093 0loss/SRqqt1/detection_loss‚1.062842299099429 trainer/global_step‚4049 2loss/SRqqt2/assignment_loss‚1.6974655091762543 1loss/FBt2/assignment_loss‚0.07028920226616701 /loss/FBt2/detection_loss‚1.5444584559493957 %loss/total_loss‚23.50558460927537 0loss/FRt1/assignment_loss‚3.6522268941325526 0loss/FBt1/assignment_loss‚0.6499648609381742 1loss/SRqqt2/detection_loss‚2.3103531333116383 _step‚323 /loss/FRt2/assignment_loss‚3.643160303936729 /loss/FRt1/detection_loss‚2.8440912106344776 .loss/FBt1/detection_loss‚1.926640250188161 epoch‚0 $ _timestamp‚1.785811975536971e+09)3ÃÈ ÿ /loss/FRt2/assignment_loss‚3.643160303936729 /loss/FRt1/detection_loss‚2.8440912106344776 .loss/FBt1/detection_loss‚1.926640250188161 epoch‚0 $ _timestamp‚1.785811975536971e+09 _runtime‚1720.726727694 1loss/SRqqt1/assignment_loss‚1.449181053030297 .loss/FRt2/detection_loss‚2.654911436612093 0loss/SRqqt1/detection_loss‚1.062842299099429 trainer/global_step‚4049 2loss/SRqqt2/assignment_loss‚1.6974655091762543 1loss/FBt2/assignment_loss‚0.07028920226616701 /loss/FBt2/detection_loss‚1.5444584559493957 %loss/total_loss‚23.50558460927537 0loss/FRt1/assignment_loss‚3.6522268941325526 0loss/FBt1/assignment_loss‚0.6499648609381742 1loss/SRqqt2/detection_loss‚2.3103531333116383 _step‚323Ã5É ‚(:… ‹¨ÅÓæÖÿ‚ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚98.667' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.889 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01ö£lÊ ‚(:Ó  ‹¨ÅÓå±Íƒ disk.drbd1226.out‚0 disk.sda1.in‚0 disk.drbd1938.out‚0$ memory_percent‚41.22898578643799 disk.drbd1673.out‚0( proc.memory.availableMB‚ 38516.171875$ disk./.usageGB‚586.3500556945801 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.drbd1289.in‚0 network.recv‚286746 disk.sda.out‚59.03125RŽTìò disk.drbd2078.in‚0 disk.sda1.out‚0 disk.drbd1226.in‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.drbd1673.in‚0 disk.drbd1356.out‚0 network.sent‚ 3.761287e+06 disk.drbd1510.out‚0 disk.rbd1.out‚ 7.0439453125 disk.drbd2041.out‚0 disk.sda.in‚ 0.4140625 disk.nvme0n1.in‚2.46875 disk.rbd0.out‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 cpu‚13.84522109482428 disk.drbd1552.out‚0 disk.rbd0.in‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1356.in‚0 disk.drbd1584.in‚0 disk.drbd2111.out‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.drbd1510.in‚0$ disk.nvme0n1.out‚599.83837890625 disk.drbd1965.in‚0) disk./.usagePercent‚32.79176462855868 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.rbd1.in‚0$ proc.memory.rssMB‚11254.03515625 proc.cpu.threads‚29 disk.drbd2078.out‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.4140625 disk.sda2.out‚59.03125 disk.drbd1584.out‚0* proc.memory.percent‚17.172294855117798 disk.drbd1965.out‚0! disk.dm-0.out‚599.83837890625 disk.drbd1552.in‚0 disk.drbd1828.in‚0Å…2øœË ‚(:‘ š¨ÅÓµšƒ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚124.867' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.62233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0—íÛì_Ì ‚(:Ô  š¨ÅÓ ‡éƒ disk.drbd1931.out‚0 disk.drbd1965.out‚0 disk.dm-0.out‚ 600.25390625 disk.drbd1673.in‚0 disk.drbd1356.out‚0 disk.drbd1550.out‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd2078.out‚0 cpu‚13.843870847120858$ disk./.usageGB‚586.3500366210938 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.rbd0.in‚0 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.rbd0.out‚0 disk.sda2.out‚ 59.42578125 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 7.1064453125 disk.drbd1584.in‚0 disk.sda.out‚ 59.42578125 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1938.out‚0) proc.memory.availableMB‚ 38515.6171875 disk.drbd1965.in‚0 disk.drbd1510.in‚0 disk.drbd1552.in‚0 disk.drbd1817.out‚0 disk.drbd2041.in‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1817.in‚0 disk.drbd1289.in‚0 disk.nvme0n1.in‚2.46875) disk./.usagePercent‚32.79176356186947 disk.drbd1285.out‚0* proc.memory.percent‚17.172294855117798 network.recv‚289364 disk.dm-0.in‚2.46875 disk.sda2.in‚ 0.4140625 disk.drbd1356.in‚0# memory_percent‚41.2298321723938$ proc.memory.rssMB‚11254.03515625 proc.cpu.threads‚29 disk.sda.in‚ 0.4140625! disk.nvme0n1.out‚ 600.25390625 network.sent‚ 3.806323e+06 disk.drbd1510.out‚0 disk.drbd2118.in‚0 disk.drbd2041.out‚0ÿÖAd2Í - _wandbruntime‚1741  _runtime‚1741Qš5ïÈÎ Â _step‚324 ' lr-AdamW/pg1‚0.00010564019240326969 ' lr-AdamW/pg2‚0.00010564019240326969 trainer/global_step‚4099 $ _timestamp‚1.785811995940548e+09 _runtime‚1741.1304981596@“ÍÏ Ç _step‚324 ' lr-AdamW/pg1‚0.00010564019240326969 ' lr-AdamW/pg2‚0.00010564019240326969 trainer/global_step‚4099 $ _timestamp‚1.785811995940548e+09 _runtime‚1741.130498159Äþ²‚2Ð - _wandbruntime‚1741  _runtime‚1741u:ÃN¾'Ñ ¸O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3704985347072 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3704854672896 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚73995637899264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚74260013465600 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27883566 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚70555028118528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11739721 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3607279456768 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚70555003214848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚4099 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3607366886400 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚72237702073548 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚68639251189472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70533690 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27240879 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3598536687200 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27883818 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚70388253973504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858119959414883e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43292813 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚70388253973504 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚73995533430272 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3607279456768 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11739471 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43302469 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚73995533430272 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚74259857887744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3598450884076 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70543348 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16144095 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200åö®Ã'Ò ½O G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43302469 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚70388271012864 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚70388271012864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚70555003214848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚4099 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3607366886400 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚72237702073548 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚68639251189472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70533690 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27240879 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3598536687200 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27883818 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚70388253973504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858119959414883e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43292813 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚70388253973504 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚73995533430272 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3607279456768 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11739471 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43302469 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚73995533430272 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚74259857887744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3598450884076 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70543348 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16144095 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3704985347072 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3704854672896 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚73995637899264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚74260013465600 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27883566 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚70555028118528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11739721 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3607279456768 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27890337 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚72255422561686 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3608246513152 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚1741.481605969 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11742305 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚74277965365248 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27890587 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3599328357298 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚4099 % _timestamp‚1.7858119962913163e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27247520 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚68656008401152 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚74013579812864 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27247522 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚70572210331648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3608159076352 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3608159076352 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27247520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11742553 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚70572235235328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚74013684289024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27247522 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43313006 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43303336 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚72255319719090 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚70405420736512 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚705605sALÑ®28 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16148032 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚68655991361792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0ùÅǘÍ&Õ ÇM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚72255422561686 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3608246513152 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚1741.481605969 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11742305 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚74277965365248 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27890587 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3599328357298 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚4099 % _timestamp‚1.7858119962913163e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27247520 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚68656008401152 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚74013579812864 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27247522 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚70572210331648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3608159076352 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3608159076352 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27247520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11742553 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚70572235235328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚74013684289024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27247522 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43313006 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43303336 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚72255319719090 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚70405420736512 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70560528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16148032 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚68655991361792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70560528 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27890337 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640Æè€2Ö - _wandbruntime‚1741  _runtime‚1741…S’× þ 0loss/SRqqt2/detection_loss‚2.089010785163288 _step‚327 2loss/SRqqt1/assignment_loss‚1.5708806911697153 $ _timestamp‚1.785811996292582e+09 0loss/FRt1/assignment_loss‚3.9182799412653995 0loss/FBt1/assignment_loss‚0.4690172347958044 1loss/FBt2/assignment_loss‚0.06733718083618594 /loss/FBt2/detection_loss‚1.4267116131634616 &loss/total_loss‚23.616533089543758 epoch‚0 _runtime‚1741.482384594 /loss/FRt2/detection_loss‚2.5550433092339095 1loss/SRqqt1/detection_loss‚0.8220896552740996 /loss/FBt1/detection_loss‚3.2520520289738974 trainer/global_step‚4099 0loss/FRt2/assignment_loss‚3.6354922416598296 2loss/SRqqt2/assignment_loss‚1.4395582205812696 .loss/FRt1/detection_loss‚2.371060187426897H>ÐÐ Ø ƒ 0loss/FRt2/assignment_loss‚3.6354922416598296 2loss/SRqqt2/assignment_loss‚1.4395582205812696 .loss/FRt1/detection_loss‚2.371060187426897 0loss/SRqqt2/detection_loss‚2.089010785163288 _step‚327 2loss/SRqqt1/assignment_loss‚1.5708806911697153 $ _timestamp‚1.785811996292582e+09 0loss/FRt1/assignment_loss‚3.9182799412653995 0loss/FBt1/assignment_loss‚0.4690172347958044 1loss/FBt2/assignment_loss‚0.06733718083618594 /loss/FBt2/detection_loss‚1.4267116131634616 &loss/total_loss‚23.616533089543758 epoch‚0 _runtime‚1741.482384594 /loss/FRt2/detection_loss‚2.5550433092339095 1loss/SRqqt1/detection_loss‚0.8220896552740996 /loss/FBt1/detection_loss‚3.2520520289738974 trainer/global_step‚4099Çæќ٠‚(:‘ ©¨ÅÓ¾ýÿ‚ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚109.144' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚36.38133333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ìC'=ZÚ ‚(:Ï  ©¨ÅÓÁ†Èƒ disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0$ proc.memory.rssMB‚11254.03515625 disk.sda2.in‚ 0.4140625 proc.cpu.threads‚29 disk.drbd1552.in‚0 disk.drbd1226.in‚0 disk.drbd1828.in‚0$ memory_percent‚41.23011827468872) proc.memory.availableMB‚ 38515.4296875* proc.memory.percent‚17.172294855117798 disk.drbd1510.in‚0 disk.drbd1356.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.dm-0.out‚ 600.40625 disk.drbd1510.out‚0 disk.drbd1828.out‚0 cpu‚13.842619174263715$ disk./.usageGB‚586.3500366210938 disk.drbd1289.out‚0 disk.drbd1552.out‚0 disk.drbd1938.out‚0) disk./.usagePercent‚32.79176356186947 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd2111.in‚0 disk.sda2.out‚ 59.73828125 disk.rbd0.out‚0 disk.dm-0.in‚2.46875 disk.nvme0n1.out‚ 600.40625 disk.drbd1356.out‚0 disk.drbd1965.out‚0 disk.drbd1550.in‚0 network.sent‚ 3.809346e+06 disk.drbd1817.in‚0 disk.drbd2111.out‚0 network.recv‚290365 disk.drbd2118.out‚0 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1226.out‚0 disk.drbd2041.out‚0 disk.sda.out‚ 59.73828125 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 7.1064453125 disk.drbd1584.out‚0 disk.sda.in‚ 0.4140625 disk.drbd1938.in‚0 disk.drbd1285.in‚0•‡>¿2Û - _wandbruntime‚1761  _runtime‚1761ô¹ŠOÉÜ Ã % _timestamp‚1.7858120167862532e+09 _runtime‚1761.976182863 _step‚328 ' lr-AdamW/pg1‚0.00010692880172753499 ' lr-AdamW/pg2‚0.00010692880172753499 trainer/global_step‚4149:dbÎÝ È % _timestamp‚1.7858120167862532e+09 _runtime‚1761.976182863 _step‚328 ' lr-AdamW/pg1‚0.00010692880172753499 ' lr-AdamW/pg2‚0.00010692880172753499 trainer/global_step‚4149È •`2Þ - _wandbruntime‚1761  _runtime‚1761øïÁ·¾'ß ¸O LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚74897972185088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43820815 _runtime‚1761.980634828 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3651363021824 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚227 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43830471 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28223315 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚71403500 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7858120167871885e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43830471 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 124382720 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚74897867716096 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚74897867716096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27573029 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11882143 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚73118701995540 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43820815 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16340945 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3642339347584 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 149286400 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚71415383958528 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚71393842 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16340947 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3750049727488 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11882368 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3651363021824 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚75165558068736 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚69476276844832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27573029 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3651275592192 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚71415359054848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚69476259805472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7858120167871885e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43830471 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 124382720 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚74897867716096 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚74897867716096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27573029 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11882143 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚73118701995540 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43820815 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16340945 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3642339347584 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 149286400 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚71415383958528 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚71393842 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16340947 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3750049727488 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11882368 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3651363021824 ]JDeviceStatsMonitor.onágu‡ _train_batch_start/inactive_split_bytes.all.allocated‚75165558068736 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚69476276844832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27573029 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3651275592192 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚71415359054848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚69476259805472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3651363021824 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚227 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43830471 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28223315 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640ÉZÌ Ÿ2á - _wandbruntime‚1762  _runtime‚1762èʦ±È&â ÂM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3643216820806 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚73136216798598 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚71432591075328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚71263775926272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 _step‚330 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71411008 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚74915914098688 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27579672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43831338 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚74916018574848 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚75183515538944 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚73136319641194 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71420680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚4149 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71411008 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚71263758886912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3652242648576 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16344882 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚71432566171648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚69492999977792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71420680 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27579672 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3652155211776 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43841008 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28229762 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚69493017017152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28230010 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3750949367296 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7858120171353853e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3652242648576 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27579670 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚71263758886912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1ƒíÄÍ&ã ÇM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43831338 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚74916018574848 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚75183515538944 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚73136319641194 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71420680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚4149 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71411008 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚71263758886912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3652242648576 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16344882 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚71432566171648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚69492999977792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71420680 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27579672 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3652155211776 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43841008 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28229762 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚69493017017152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28230010 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3750949367296 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7858120171353853e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3652242648576 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27579670 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚71263758886912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3643216820806 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚73136216798598 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚71432591075328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚71263775926272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 _step‚330 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71411008 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚74915914098688 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27579672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888ÊÆU¢@2ä - _wandbruntime‚1762  _runtime‚1762ÆšÛå þ 2loss/SRqqt1/assignment_loss‚1.4753197593460405 2loss/SRqqt2/assignment_loss‚1.3826580621302127 trainer/global_step‚4149 /loss/FRt2/detection_loss‚2.3208785589458873 2loss/FBt1/assignment_loss‚0.042432540612935554 1loss/FBt2/assignment_loss‚0.16807983677052032 .loss/FRt1/detection_loss‚2.331880339943781 1loss/SRqqt1/detection_loss‚0.9965925899589885 0loss/SRqqt2/detection_loss‚2.253943942238887 0loss/FRt2/assignment_loss‚3.7583597724877515 .loss/FBt1/detection_loss‚20.34312127654751 /loss/FBt2/detection_loss‚1.7589633198066925 %loss/total_loss‚40.37079551962238 % _timestamp‚1.7858120171365993e+09 _runtime‚1762.326388841 _step‚331 epoch‚0 /loss/FRt1/assignment_loss‚3.538565520833178AcÑ æ ƒ 0loss/FRt2/assignment_loss‚3.7583597724877515 .loss/FBt1/detection_loss‚20.34312127654751 /loss/FBt2/detection_loss‚1.7589633198066925 %loss/total_loss‚40.37079551962238 % _timestamp‚1.7858120171365993e+09 _runtime‚1762.326388841 _step‚331 epoch‚0 /loss/FRt1/assignment_loss‚3.538565520833178 2loss/SRqqt1/assignment_loss‚1.4753197593460405 2loss/SRqqt2/assignment_loss‚1.3826580621302127 trainer/global_step‚4149 /loss/FRt2/detection_loss‚2.3208785589458873 2loss/FBt1/assignment_loss‚0.042432540612935554 1loss/FBt2/assignment_loss‚0.16807983677052032 .loss/FRt1/detection_loss‚2.331880339943781 1loss/SRqqt1/detection_loss‚0.9965925899589885 0loss/SRqqt2/detection_loss‚2.253943942238887Ë™hë­šç ‚(: ¸¨ÅÓÕ𫃠gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚96.44' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.14666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0C^Igè ‚(:Ü  ¸¨ÅÓðÜàƒ disk.drbd1938.in‚0 disk.drbd1931.in‚0 disk.dm-0.in‚2.46875 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1285.in‚0 proc.cpu.threads‚29* proc.memory.percent‚17.172294855117798 disk.drbd1289.in‚0 disk.drbd1828.in‚0% memory_percent‚41.230183839797974 disk.sda1.in‚0 disk.drbd2118.in‚0 disk.rbd1.out‚ 7.138671875 disk.drbd1965.in‚0 network.sent‚ 3.854343e+06 disk.drbd1938.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0! disk.dm-0.out‚600.87744140625* proc.memory.availableMB‚38515.38671875 disk.drbd1817.in‚0) disk./.usagePercent‚32.79176676193709$ disk./.usageGB‚586.3500938415527 disk.drbd2118.out‚0 network.recv‚292878 disk.drbd1550.in‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.drbd1584.out‚0 disk.drbd2078.out‚0 disk.drbd1552.in‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.drbd2111.out‚0 disk.rbd0.out‚0 disk.sda2.in‚ 0.4140625 disk.sda2.out‚ 60.01171875 disk.sda.in‚ 0.4140625$ proc.memory.rssMB‚11254.03515625 disk.drbd1510.in‚0 cpu‚13.840971757264342 disk.rbd0.in‚0 disk.drbd2111.in‚0 disk.sda.out‚ 60.01171875 disk.drbd1226.in‚0 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.rbd1.in‚0$ disk.nvme0n1.out‚600.87744140625 disk.drbd1552.out‚0ª‹Âa2é - _wandbruntime‚1783  _runtime‚1783gc’VÇê Á _runtime‚1783.036411527 _step‚332 & lr-AdamW/pg1‚0.0001082174110518003 & lr-AdamW/pg2‚0.0001082174110518003 trainer/global_step‚4199 % _timestamp‚1.7858120378464792e+09÷ YÌë Æ _step‚332 & lr-AdamW/pg1‚0.0001082174110518003 & lr-AdamW/pg2‚0.0001082174110518003 trainer/global_step‚4199 % _timestamp‚1.7858120378464792e+09 _runtime‚1783.036411527Ì6i¾2ì - _wandbruntime‚1783  _runtime‚1783icí ¸O VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚72104930274304 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚75800306470912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚72104947313664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3695359157248 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76071110223360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16537795 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72253994 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44358473 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚70313285460832 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚75800202001920 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚75800202001920 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3695271727616 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚72104947313664 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44358473 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚72275739798528 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44348817 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3695271727616 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12024835 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚70313268421472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 % _timestamp‚1.7858120378473887e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚72275714894848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27905177 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚72104930274304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3686227811092 XEDeviceStatsMonitor.on_train_batch_start˜¢Ca¥/allocated_bytes.all.allocated‚75800306470912 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27905179 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16537797 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3686313614216 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3795370424832 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72263652 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _runtime‚1783.040624882 _step‚333Žp†ˆÃ'î ½O KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12024581 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28562376 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚73999599075048 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚76070958839808 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3795370424832 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72263652 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _runtime‚1783.040624882 _step‚333 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚72104930274304 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚75800306470912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚72104947313664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3695359157248 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76071110223360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16537795 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72253994 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44358473 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚70313285460832 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚75800202001920 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚75800202001920 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3695271727616 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚72104947313664 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44358473 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚72275739798528 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44348817 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3695271727616 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12024835 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚70313268421472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 % _timestamp‚1.7858120378473887e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚72275714894848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27905177 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚72104930274304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3686227811092 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚75800306470912 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27905179 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16537797 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3686313614216 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44369010 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚76089067365888 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27911822 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚72292922011648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚72122097037312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3696238784000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚4199 % _timestamp‚1.7858120382002375e+09 _runtime‚ 1783.39054565 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚72122114076672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74017216720702 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27911820 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚72292946915328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚334 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3687105284314 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27911822 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚70330025633152 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚72122097037312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28569187 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12027455 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3696151347200 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3687191087550 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44359340 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28569446 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3696151347200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44359340 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚72271160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72280832 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚76089225033728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚75818352860672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3696238784000 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72280832 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44369010 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚76089067365888 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27911822 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚72292922011648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚72122097037312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3696238784000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚4199 % _timestamp‚1.7858120382002375e+09 _runtime‚ 1783.39054565 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚72122114076672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74017216720702 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27911820 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚72292946915328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚334 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3687105284314 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27911822 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚70330025633152 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚72122097037312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28569187 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12027455 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3696151347200 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3687191087550 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44359340 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28569446 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsühÑPMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3696151347200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44359340 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚72271160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72280832 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚76089225033728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚75818352860672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3696238784000 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72280832 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856νM2ò - _wandbruntime‚1783  _runtime‚1783ŠTÂó ü 2loss/SRqqt2/assignment_loss‚1.3570073458930136 1loss/SRqqt1/detection_loss‚0.8540857064189308 /loss/FBt1/detection_loss‚2.7160897254943848 /loss/FRt2/assignment_loss‚3.249041027181289 2loss/SRqqt1/assignment_loss‚1.6350878477096558 /loss/FBt1/assignment_loss‚0.657831747829914 /loss/FRt2/detection_loss‚2.6010340150664835 0loss/SRqqt2/detection_loss‚2.040762281124021 &loss/total_loss‚24.511767921397748 trainer/global_step‚4199 % _timestamp‚1.7858120382014515e+09 /loss/FRt1/assignment_loss‚4.158778345430052 1loss/FBt2/assignment_loss‚0.14556845696579918 /loss/FRt1/detection_loss‚2.6235763713136895 epoch‚0 _runtime‚1783.391255766 _step‚335 .loss/FBt2/detection_loss‚2.472905050970513Õÿ¥ô  /loss/FRt2/assignment_loss‚3.249041027181289 2loss/SRqqt1/assignment_loss‚1.6350878477096558 /loss/FBt1/assignment_loss‚0.657831747829914 /loss/FRt2/detection_loss‚2.6010340150664835 0loss/SRqqt2/detection_loss‚2.040762281124021 &loss/total_loss‚24.511767921397748 trainer/global_step‚4199 % _timestamp‚1.7858120382014515e+09 /loss/FRt1/assignment_loss‚4.158778345430052 1loss/FBt2/assignment_loss‚0.14556845696579918 /loss/FRt1/detection_loss‚2.6235763713136895 epoch‚0 _runtime‚1783.391255766 _step‚335 .loss/FBt2/detection_loss‚2.472905050970513 2loss/SRqqt2/assignment_loss‚1.3570073458930136 1loss/SRqqt1/detection_loss‚0.8540857064189308 /loss/FBt1/detection_loss‚2.7160897254943848Ï\·Äœõ ‚(:‘ ǨÅÓ™Áƒ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚122.312' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.77066666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$ðøÍbö ‚(:×  ǨÅÓâ˜Ìƒ) disk./.usagePercent‚32.79176676193709 disk.sda.in‚ 0.4140625* proc.memory.percent‚17.172294855117798 disk.drbd1584.in‚0 disk.drbd1226.in‚0 disk.drbd1673.out‚0 disk.sda2.in‚ 0.4140625 network.sent‚ 3.89931e+06 disk.drbd1828.in‚0 disk.drbd2118.out‚0$ disk./.usageGB‚586.3500938415527 disk.drbd1550.out‚0 disk.dm-0.in‚2.46875 disk.drbd2111.out‚0 disk.drbd1673.in‚0 cpu‚13.839424232663095 proc.cpu.threads‚29 disk.drbd1226.out‚0 disk.drbd1817.in‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd2078.in‚0 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.drbd1356.out‚0 disk.drbd1552.in‚0 disk.drbd1938.out‚0 disk.drbd1356.in‚0 disk.drbd1550.in‚0 disk.drbd1931.in‚0 disk.drbd1965.out‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd1965.in‚0 disk.drbd2111.in‚0) proc.memory.availableMB‚ 38515.3203125 disk.nvme0n1.in‚2.46875! disk.dm-0.out‚601.01806640625 disk.drbd1552.out‚0 disk.drbd2118.in‚0 network.recv‚295270 disk.drbd2041.out‚0 disk.drbd1285.out‚0 disk.drbd1289.in‚0 disk.drbd1817.out‚0 disk.sda2.out‚ 60.3671875 disk.drbd2041.in‚0 disk.drbd1285.in‚0 disk.sda1.in‚0$ disk.nvme0n1.out‚601.01806640625$ memory_percent‚41.23028516769409 disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.rbd1.out‚ 7.318359375 disk.drbd1584.out‚0 disk.sda.out‚ 60.3671875 disk.rbd0.out‚0 disk.drbd1510.in‚0 disk.drbd1938.in‚0YSR=œ÷ ‚(:‘ Ö¨ÅÓç ¦ƒ gpu.0.gpu‚34.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚109.03' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚36.343333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0§)¢‚fø ‚(:Û  Ö¨ÅÓîÉÓƒ disk.drbd1938.in‚0 disk.drbd1510.in‚0 disk.rbd1.out‚ 7.380859375$ disk.nvme0n1.out‚601.42822265625 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 network.recv‚297666 disk.drbd1550.out‚0 disk.drbd1289.in‚0 proc.cpu.threads‚29 network.sent‚ 3.944302e+06 disk.drbd1817.in‚0 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1828.in‚0* disk./.usagePercent‚32.791748201544934 disk.drbd1226.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0$ proc.memory.rssMB‚11254.03515625* proc.memory.percent‚17.172294855117798 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.sda.out‚ 60.99609375 disk.drbd2078.in‚0 disk.drbd1828.out‚0$ memory_percent‚41.22990369796753 disk.dm-0.in‚2.46875! disk.dm-0.out‚601.42822265625 disk.drbd1552.out‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd1285.out‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd2041.out‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd1550.in‚0 disk.drbd1510.out‚0$ disk./.usageGB‚586.3497619628906 disk.drbd1356.out‚0 disk.sda.in‚ 0.4140625 disk.drbd2118.out‚0 disk.sda2.in‚ 0.4140625 disk.drbd2111.in‚0 disk.drbd1584.in‚0 disk.drbd1965.out‚0 disk.drbd2111.out‚0 disk.drbd1356.in‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 60.99609375 disk.nvme0n1.in‚2.46875 disk.drbd2078.out‚0) proc.memory.availableMB‚ 38515.5703125 cpu‚13.838032433984235Ô‹Ã2ù - _wandbruntime‚1803  _runtime‚18038ÃB Èú  ' lr-AdamW/pg1‚0.00010950602037606561 ' lr-AdamW/pg2‚0.00010950602037606561 trainer/global_step‚4249 % _timestamp‚1.7858120583335967e+09 _runtime‚ 1803.52354328 _step‚336£j$»Íû Ç ' lr-AdamW/pg1‚0.00010950602037606561 ' lr-AdamW/pg2‚0.00010950602037606561 trainer/global_step‚4249 % _timestamp‚1.7858120583335967e+09 _runtime‚ 1803.52354328 _step‚336ÐH;2ü - _wandbruntime‚1803  _runtime‚1803Ô`¾'ý ¸O SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12167325 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76976668396544 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3739355292672 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16734647 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚71150294076832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73123804 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚73136095638528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44876819 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28237327 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚76702536287744 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3739355292672 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚76976512818688 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3840442083840 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28901972 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚74880496154556 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73114146 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3739267863040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚72963285464064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3739267863040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16734645 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12167057 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚73136070734848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚72963285464064 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44876819 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28237327 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3840572758016 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚76702536287744 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚76702640756736 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3730116274600 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28237329 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚72963268424704 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚72963268424704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2§r¤Ã'þ ½O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28237329 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚72963268424704 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚72963268424704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12167325 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76976668396544 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3739355292672 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16734647 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚71150294076832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73123804 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚73136095638528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44876819 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28237327 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚76702536287744 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3739355292672 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚76976512818688 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3840442083840 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28901972 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚74880496154556 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73114146 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3739267863040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚72963285464064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3739267863040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16734645 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12167057 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚73136070734848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚72963285464064 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44876819 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28237327 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3840572758016 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚76702536287744 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚76702640756736 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3730116274600ÑDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28908412 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚338 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3740147482624 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚76720687146496 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3841350629888 C6DeviceStatsMonitor.on¸1ÉÄn_train_batch_end/active.all.freed‚73131312 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚76720582670336 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12170061 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚71167017209792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28243970 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73140984 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73131312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚233 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73140984 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28243970 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚72980452227072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3740234919424 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16738582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3731079551058 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚72980435187712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3841475005440 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚76994628481536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚73153277851648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28243972 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74898113800210 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16738584 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 $ _timestamp‚1.785812058685506e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚73153302755328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44897012 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚4249 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360ˆstšÌ& ÆM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28243972 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74898113800210 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16738584 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 $ _timestamp‚1.785812058685506e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚73153302755328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44897012 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚4249 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28243972 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44887342 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44887342 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚72980452227072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚71167034249152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3740147482624 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28908412 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚338 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3740147482624 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚76720687146496 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3841350629888 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73131312 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚76720582670336 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12170061 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚71167017209792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28243970 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73140984 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73131312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚233 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73140984 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28243970 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚72980452227072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3740234919424 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16738582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3731079551058 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚72980435187712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3841475005440 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚76994628481536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚73153277851648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936Ò[&å2‚ - _wandbruntime‚1803  _runtime‚1803·.g¦ƒ ý 1loss/SRqqt2/detection_loss‚2.2656135228726098 /loss/FBt1/detection_loss‚2.6990341141187986 /loss/FBt2/detection_loss‚1.8258429378537195 _runtime‚1803.876455552 _step‚339 2loss/SRqqt1/assignment_loss‚1.4781933196112513 /loss/FRt1/detection_loss‚2.4582158375531433 1loss/SRqqt1/detection_loss‚0.9848007469180712 0loss/FBt1/assignment_loss‚0.3109844010276512 epoch‚0 /loss/FRt1/assignment_loss‚3.809625582396984 2loss/SRqqt2/assignment_loss‚1.6137347395689983 /loss/FRt2/detection_loss‚2.3264917787752655 &loss/total_loss‚23.553395900700817 trainer/global_step‚4249 % _timestamp‚1.7858120586866963e+09 0loss/FRt2/assignment_loss‚3.6690479604821458 .loss/FBt2/assignment_loss‚0.11181095952218Pú„ ‚ &loss/total_loss‚23.553395900700817 trainer/global_step‚4249 % _timestamp‚1.7858120586866963e+09 0loss/FRt2/assignment_loss‚3.6690479604821458 .loss/FBt2/assignment_loss‚0.11181095952218 1loss/SRqqt2/detection_loss‚2.2656135228726098 /loss/FBt1/detection_loss‚2.6990341141187986 /loss/FBt2/detection_loss‚1.8258429378537195 _runtime‚1803.876455552 _step‚339 2loss/SRqqt1/assignment_loss‚1.4781933196112513 /loss/FRt1/detection_loss‚2.4582158375531433 1loss/SRqqt1/detection_loss‚0.9848007469180712 0loss/FBt1/assignment_loss‚0.3109844010276512 epoch‚0 /loss/FRt1/assignment_loss‚3.809625582396984 2loss/SRqqt2/assignment_loss‚1.6137347395689983 /loss/FRt2/detection_loss‚2.3264917787752655Ó‹´^‘… ‚(:† å¨ÅÓ›Öžƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.524' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.508 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0•ø¥€c† ‚(:Ø  å¨ÅÓìüæƒ disk.rbd1.in‚0) proc.memory.availableMB‚ 38515.5390625 network.recv‚298603) disk./.usagePercent‚32.79180110932947 disk.drbd2041.out‚0 disk.sda.out‚ 61.3046875 disk.drbd1673.out‚0 disk.sda2.out‚ 61.3046875 disk.drbd1965.out‚0 disk.rbd0.out‚0 disk.drbd1584.in‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1673.in‚0$ disk.nvme0n1.out‚601.70947265625 disk.drbd1584.out‚0 disk.drbd1226.in‚0 disk.drbd1289.in‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0$ memory_percent‚41.22995138168335 proc.cpu.threads‚29 network.sent‚ 3.947335e+06 disk.sda.in‚ 0.4140625! disk.dm-0.out‚601.70947265625 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1356.in‚0 disk.drbd2078.out‚0 disk.drbd1510.in‚0$ proc.memory.rssMB‚11254.03515625* proc.memory.percent‚17.172294855117798 disk.drbd1931.out‚0 disk.rbd0.in‚0 disk.drbd1356.out‚0$ disk./.usageGB‚586.3507080078125 disk.drbd1550.out‚0 disk.sda1.in‚0 disk.drbd2078.in‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.drbd1817.in‚0 disk.drbd1965.in‚0 disk.drbd1285.in‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0 cpu‚13.836927610329747 disk.drbd1931.in‚0 disk.drbd1828.in‚0 disk.drbd1938.out‚0 disk.rbd1.out‚ 7.380859375 disk.drbd1226.out‚0 disk.drbd1828.out‚0 disk.dm-0.in‚2.46875 disk.drbd1510.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.4140625Ç_“2‡ - _wandbruntime‚1824  _runtime‚1824L¦õâɈ à ' lr-AdamW/pg2‚0.00011079462970033092 trainer/global_step‚4299 % _timestamp‚1.7858120793362086e+09 _runtime‚1824.526146182 _step‚340 ' lr-AdamW/pg1‚0.00011079462970033092@”²?Ή È % _timestamp‚1.7858120793362086e+09 _runtime‚1824.526146182 _step‚340 ' lr-AdamW/pg1‚0.00011079462970033092 ' lr-AdamW/pg2‚0.00011079462970033092 trainer/global_step‚4299ÔŠ3í2Š - _wandbruntime‚1824  _runtime‚1824å>"º½'‹ ·O ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16931497 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚262 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚71987285653472 trainer/global_step‚4299 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73974298 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚73821606575104 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16931495 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 $ _timestamp‚1.785812079337107e+09 _runtime‚1824.530629506 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚77882201212416 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚77604870573568 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45414477 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45404821 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3783351428096 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29240920 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚73996451478528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚77604975042560 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3783263998464 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚73996426574848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28569479 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3783351428096 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚73821606575104 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚77604870573568 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73983956 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚73821623614464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚73821623614464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12309687 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3774090541232 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12309425 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚75761393234064 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚341 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3774004738108 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚77882039343104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚71987302692832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121%ÿéÈ‚ Œ ¼O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73974298 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚73821606575104 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16931495 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 $ _timestamp‚1.785812079337107e+09 _runtime‚1824.530629506 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚77882201212416 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚77604870573568 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45414477 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45404821 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3783351428096 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29240920 dQDeviceStatsMonitor.on_tÄO¬@rain_batch_start/inactive_split_bytes.large_pool.allocated‚73996451478528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚77604975042560 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3783263998464 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚73996426574848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28569479 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3783351428096 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚73821606575104 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚77604870573568 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73983956 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚73821623614464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚73821623614464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12309687 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3774090541232 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12309425 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚75761393234064 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚341 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3774004738108 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚77882039343104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚71987302692832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16931497 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚262 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚71987285653472 trainer/global_step‚4299Õqòy÷2 - _wandbruntime‚1824  _runtime‚1824Š£¡XÆ&Ž ÀM I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28576122 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12312314 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚75778908037122 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 # _timestamp‚1.78581207968347e+09 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74001136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12312569 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _step‚342 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚72004042865152 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3784143618048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73991464 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28576120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45415344 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3784143618048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29248003 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚75779010879718 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16935432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3886519791104 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74001136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29247746 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚73838790377472 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚77623021432320 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3784231054848 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28576122 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 # _timestamp‚1.78581207968347e+09 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74001136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12312569 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _step‚342 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚72004042865152 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3784143618048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73991464 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28576120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45415344 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3784143618048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29248003 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚75779010879718 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16935432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3886519791104 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74001136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29247746 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚73838790377472 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚77623021432320 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3784231054848 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28576122 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45932823 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3930898245632 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚76642290313572 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3827260133888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚235 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45932823 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚78507204859392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17128347 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3930761280000 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚78507309328384 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12453884 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3827347563520 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45942479 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚78787543694848 % _timestamp‚1.7858121000825047e+09 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚78787705564160 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28901627 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28901629 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29582229 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚74856807318528 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3817893201616 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28901627 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12454117 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _step‚345 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚74679944725504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚74679944725504 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3827347563520 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚74679961764864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28901629 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251` MÃ'œ ½O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28901627 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12454117 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _step‚345 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚74679944725504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚74679944725504 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3827347563520 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚74679961764864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28901629 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚78507204859392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚4349 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚72824294269472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74844108 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚72824311308832 _runtime‚1845.273711309 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3827260133888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17128345 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚233 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45942479 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚76642187471088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚74856782414848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45932823 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3930898245632 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚76642290313572 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3827260133888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚235 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45932823 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚78507204859392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17128347 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3930761280000 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚78507309328384 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12453884 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3827347563520 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45942479 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚78787543694848 % _timestamp‚1.7858121000825047e+09 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚78787705564160 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28901627 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28901629 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29582229 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚74856807318528 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3817893201616Ùœ‡ÁÝ2 - _wandbruntime‚1845  _runtime‚1845=ûG×È&ž ÂM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3828139753472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28908270 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28908270 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3931816244736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3818856478074 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74861288 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚74697128527872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12456777 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚78525355718144 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29589059 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚78805666720768 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45943346 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚74873989531648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28908272 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3828139753472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚76659805116630 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74861288 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45953016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28908272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚74874014435328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚76659907959226 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚72841034441792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3818770674838 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 B¼Ÿ ÇM ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚72841051481152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚78525355718144 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚™iH"29589059 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚78805666720768 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45943346 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚74873989531648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28908272 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3828139753472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚76659805116630 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74861288 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45953016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28908272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚74874014435328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚76659907959226 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚72841034441792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3818770674838 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28908270 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3931816244736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3818856478074 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74861288 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚74697128527872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12456777 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3871343698944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12598449 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚73661302885472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚79409539145216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚75717163158528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3871256269312 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29233777 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3871256269312 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29233779 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚79409643614208 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46460825 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12598205 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚79693211081216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3976047922688 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚75717138254848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3861867468248 % _timestamp‚1.7858121208805013e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17325195 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18k üy£%ª ½O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46470481 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚75538299915264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29233777 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚77523187393080 _step‚349 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚75538282875904 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚75538282875904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚79409643614208 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46460825 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12598205 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚79693211081216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3976047922688 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚75717138254848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3861867468248 % _timestamp‚1.7858121208805013e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17325195 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3871343698944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12598449 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚73661302885472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚79409539145216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚75717163158528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3871256269312 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29233777 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3871256269312 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29233779 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29930147 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75721440 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚77540702196138 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75721440 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12601015 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46481018 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3872223325696 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29240420 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3862744941582 _runtime‚1866.425346718 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29240422 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚75555449638912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46471348 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚75734370275328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29240420 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚79711319976960 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚75734345371648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚228 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚230 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚75711768 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚79427690003968 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75711768 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12601243 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3872135888896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29930377 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚75555466678272 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚75555449638912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3976949701632 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4399 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚79427585527808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29240422 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3862659138346 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0j`mÌ&­ ÆM R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚77540702196138 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75721440 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12601015 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46481018 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3872223325696 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29240420 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3862744941582 _runtime‚1866.425346718 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29240422 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚75555449638912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46471348 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚75734370275328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29240420 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚79711319976960 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚75734345371648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚228 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚230 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚75711768 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚79427690003968 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75711768 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12601243 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3872135888896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29930377 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚75555466678272 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚75555449638912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3976949701632 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4399 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚79427585527808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29240422 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3862659138346 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29930147 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75721440 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024Þ¡í»2® - _wandbruntime‚1866  _runtime‚1866 Iu¯ ü /loss/FBt2/detection_loss‚1.8545231908748974 1loss/SRqqt1/assignment_loss‚1.534858962734237 2loss/SRqqt2/assignment_loss‚1.2899048305474794 1loss/SRqqt1/detection_loss‚0.7862209290756162 _runtime‚1866.426318131 /loss/FRt2/detection_loss‚2.5118261599677734 trainer/global_step‚4399 % _timestamp‚1.7858121212365332e+09 _step‚351 0loss/SRqqt2/detection_loss‚2.105417495278212 .loss/FBt1/detection_loss‚2.850139663865169 &loss/total_loss‚23.950161187750687 /loss/FRt1/assignment_loss‚4.333619255158636 0loss/FRt2/assignment_loss‚3.7119335881594955 0loss/FBt1/assignment_loss‚0.1884651005627044 1loss/FBt2/assignment_loss‚0.08475701920407724 .loss/FRt1/detection_loss‚2.698494992322392 epoch‚0³’ie°  /loss/FRt2/detection_loss‚2.5118261599677734 trainer/global_step‚4399 % _timestamp‚1.7858121212365332e+09 _step‚351 0loss/SRqqt2/detection_loss‚2.105417495278212 .loss/FBt1/detection_loss‚2.850139663865169 &loss/total_loss‚23.950161187750687 /loss/FRt1/assignment_loss‚4.333619255158636 0loss/FRt2/assignment_loss‚3.7119335881594955 0loss/FBt1/assignment_loss‚0.1884651005627044 1loss/FBt2/assignment_loss‚0.08475701920407724 .loss/FRt1/detection_loss‚2.698494992322392 epoch‚0 /loss/FBt2/detection_loss‚1.8545231908748974 1loss/SRqqt1/assignment_loss‚1.534858962734237 2loss/SRqqt2/assignment_loss‚1.2899048305474794 1loss/SRqqt1/detection_loss‚0.7862209290756162 _runtime‚1866.426318131ß‚H± ‚(:‘ ¡©ÅÓîò•ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚94.504' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚31.501333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¶]ïm]² ‚(:Ò  ¡©ÅÓݫ؃ disk.drbd2118.in‚0 network.sent‚ 4.085308e+06) disk./.usagePercent‚32.79180153600515 disk.sda2.in‚ 0.4140625 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.sda1.out‚0$ memory_percent‚41.23053550720215 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd1965.in‚0 disk.sda1.in‚0 disk.drbd1673.in‚0 proc.cpu.threads‚29 disk.sda2.out‚ 63.234375 disk.drbd1356.in‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.dm-0.out‚604.1162109375* proc.memory.percent‚17.172294855117798# disk./.usageGB‚586.350715637207 disk.drbd1965.out‚0 disk.drbd2118.out‚0$ proc.memory.rssMB‚11254.03515625 disk.drbd1510.out‚0 disk.drbd1673.out‚0 cpu‚13.830884262976463 disk.drbd1817.out‚0 disk.drbd1938.out‚0# disk.nvme0n1.out‚604.1162109375 disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd2111.out‚0 disk.drbd1289.out‚0 network.recv‚306948 disk.rbd1.out‚ 7.6630859375 disk.drbd1584.in‚0 disk.drbd1550.in‚0 disk.dm-0.in‚2.46875 disk.drbd2041.out‚0' proc.memory.availableMB‚ 38515.15625 disk.sda.in‚ 0.4140625 disk.drbd1552.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1285.out‚0 disk.drbd1828.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd2041.in‚0 disk.drbd1552.out‚0 disk.drbd1584.out‚0 disk.sda.out‚ 63.234375°ïd¼2³ - _wandbruntime‚1887  _runtime‚1887òlglÈ´  _step‚352 ' lr-AdamW/pg1‚0.00011466045767312683 ' lr-AdamW/pg2‚0.00011466045767312683 trainer/global_step‚4449 $ _timestamp‚1.785812141981008e+09 _runtime‚1887.170958378qÄë͵ Ç ' lr-AdamW/pg1‚0.00011466045767312683 ' lr-AdamW/pg2‚0.00011466045767312683 trainer/global_step‚4449 $ _timestamp‚1.785812141981008e+09 _runtime‚1887.170958378 _step‚352à,vÏc2¶ - _wandbruntime‚1887  _runtime‚1887$É>Œ· ¸O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29565929 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚80311873431040 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29565929 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29565927 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚78404084472588 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _runtime‚1887.175641291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46998483 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4021094370304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚76396621026304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚76564412 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚80311977900032 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚80311873431040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30262477 trainer/global_step‚4449 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚76554754 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12740676 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3905670128632 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17522047 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46988827 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3905755931756 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚76577518998528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚74498328540832 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚80598588465152 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚76577494094848 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29565927 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚76396621026304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚353 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3915339834368 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30262723 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17522045 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4021231335936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7858121419819632e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚78403981630104 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/sžÆ,S2egment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚76396638065664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3915252404736 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚80598750334464 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚76396638065664 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚80311977900032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12740432 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46998483 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46988827 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3915339834368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360úõKûÃ'¸ ½O MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12740432 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46998483 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46988827 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3915339834368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29565929 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚80311873431040 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29565929 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29565927 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚78404084472588 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _runtime‚1887.175641291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46998483 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4021094370304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚76396621026304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚76564412 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚80311977900032 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚80311873431040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30262477 trainer/global_step‚4449 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚76554754 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12740676 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3905670128632 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17522047 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46988827 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3905755931756 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚76577518998528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚74498328540832 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚80598588465152 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚76577494094848 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29565927 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚76396621026304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚353 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3915339834368 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30262723 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17522045 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4021231335936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7858121419819632e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚78403981630104 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚76396638065664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3915252404736 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚80598750334464 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚76396638065664 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚80311977900032áÉÚBØ2¹ - _wandbruntime‚1887  _runtime‚1887<^&È&º ÂM J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30269219 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47009020 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚80330024289792 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76581592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12743237 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚76413787789312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12743489 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚80616857427968 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3916219461120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30269473 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 % _timestamp‚1.7858121423324025e+09 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76571920 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚76594701211648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚4449 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76571920 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚74515068713152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3906547601854 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚80616697662976 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚76594726115328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3916132024320 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29572570 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 _runtime‚1887.522672189ÂõpEÍ&» ÇM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12743489 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚80616857427968 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3916219461120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30269473 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 % _timestamp‚1.7858121423324025e+09 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76571920 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚76594701211648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚4449 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76571920 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚74515068713152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3906547601854 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚80616697662976 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚76594726115328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3916132024320 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29572570 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 _runtime‚1887.522672189 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30269219 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47009020 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚80330024289792 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76581592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12743237 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚76413787789312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712âUCé2¼ - _wandbruntime‚1887  _runtime‚1887«ÿ˜(á½ € .loss/FBt1/detection_loss‚1.803078911521218 trainer/global_step‚4449 0loss/FRt2/assignment_loss‚3.5114697573032787 0loss/FBt1/assignment_loss‚0.8260012633558786 /loss/FRt2/detection_loss‚2.2891185182206173 1loss/SRqqt2/detection_loss‚2.1848618193499503 % _timestamp‚1.7858121423336084e+09 _runtime‚1887.523379579 0loss/FRt1/assignment_loss‚3.9780602135309358 1loss/FBt2/assignment_loss‚0.07092413441782391 /loss/FRt1/detection_lossììd%‚2.4254526341470277 _step‚355 &loss/total_loss‚23.172954883112865 epoch‚0 2loss/SRqqt1/assignment_loss‚1.5673072873523421 2loss/SRqqt2/assignment_loss‚1.7279055974175852 1loss/SRqqt1/detection_loss‚1.3103579408136978 /loss/FBt2/detection_loss‚1.4784168056825078v×V ¾ … .loss/FBt1/detection_loss‚1.803078911521218 trainer/global_step‚4449 0loss/FRt2/assignment_loss‚3.5114697573032787 0loss/FBt1/assignment_loss‚0.8260012633558786 /loss/FRt2/detection_loss‚2.2891185182206173 1loss/SRqqt2/detection_loss‚2.1848618193499503 % _timestamp‚1.7858121423336084e+09 _runtime‚1887.523379579 0loss/FRt1/assignment_loss‚3.9780602135309358 1loss/FBt2/assignment_loss‚0.07092413441782391 /loss/FRt1/detection_loss‚2.4254526341470277 _step‚355 &loss/total_loss‚23.172954883112865 epoch‚0 2loss/SRqqt1/assignment_loss‚1.5673072873523421 2loss/SRqqt2/assignment_loss‚1.7279055974175852 1loss/SRqqt1/detection_loss‚1.3103579408136978 /loss/FBt2/detection_loss‚1.4784168056825078ã¤ék4`¿ ‚(:Õ  °©ÅÓ™ÞŃ disk.dm-0.in‚2.46875" proc.memory.rssMB‚ 11254.140625 disk.sda.in‚ 0.4140625 disk.drbd1938.out‚0 disk.sda2.out‚ 63.83203125 disk.drbd1356.in‚0 disk.drbd2078.out‚0 disk.drbd1289.out‚0 disk.drbd1226.in‚0 disk.drbd1510.out‚0 disk.sda1.in‚0 disk.rbd1.out‚ 7.7255859375) disk./.usagePercent‚32.79180153600515 disk.drbd1356.out‚0 disk.drbd1817.out‚0 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚2.46875 disk.rbd0.in‚0 network.sent‚ 4.130496e+06 disk.drbd1552.in‚0 disk.rbd0.out‚0 disk.drbd2111.out‚0# disk./.usageGB‚586.350715637207 disk.rbd1.in‚0$ memory_percent‚41.23079776763916 cpu‚13.82994521587215 disk.drbd1226.out‚0 disk.sda2.in‚ 0.4140625 disk.drbd1965.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 proc.cpu.threads‚29 disk.drbd1552.out‚0 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.drbd1550.out‚0 disk.drbd2111.in‚0 network.recv‚309370 disk.sda.out‚ 63.83203125 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.drbd2118.out‚0( proc.memory.availableMB‚ 38514.984375 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd1965.out‚0) proc.memory.percent‚17.17245578765869$ disk.nvme0n1.out‚604.45556640625 disk.drbd1285.out‚0 disk.drbd1931.out‚0 disk.drbd1285.in‚0! disk.dm-0.out‚604.45556640625 disk.sda1.out‚0 disk.drbd1817.in‚0—p1ËÀ ‚(:… °©ÅÓó–̓ gpu.0.gpu‚39.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚94.899' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚31.633 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0QãÇsœÁ ‚(:‘ ¿©ÅÓ¦ˆ¦ƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚124.664' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.55466666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0c€hŠ_ ‚(:Ô  ¿©ÅÓðÞრdisk.drbd1584.in‚0 disk.drbd1828.in‚0 cpu‚13.828252916712186 proc.cpu.threads‚29 disk.drbd1289.out‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0$ memory_percent‚41.23120307922363) disk./.usagePercent‚32.79180046931594 disk.sda2.in‚ 0.4140625 disk.drbd1356.in‚0 disk.drbd2041.out‚0 disk.drbd2078.out‚0 network.sent‚ 4.175545e+06$ disk./.usageGB‚586.3506965637207 disk.drbd1673.out‚0 disk.drbd1965.in‚0 disk.drbd1550.in‚0! disk.dm-0.out‚604.67041015625 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd1550.out‚0 network.recv‚311905 disk.drbd1673.in‚0 disk.sda2.out‚ 64.1328125 disk.rbd1.out‚ 7.8193359375 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.rbd1.in‚0 disk.drbd1226.in‚0 disk.drbd1828.out‚0" proc.memory.rssMB‚ 11254.140625 disk.sda1.out‚0 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.drbd1226.out‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚2.46875 disk.sda.out‚ 64.1328125 disk.rbd0.out‚0 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.drbd2041.in‚0 disk.sda.in‚ 0.4140625' proc.memory.availableMB‚ 38514.71875 disk.drbd1289.in‚0 disk.drbd1938.out‚0 disk.drbd1931.out‚0 disk.dm-0.in‚2.46875) proc.memory.percent‚17.17245578765869 disk.sda1.in‚0 disk.drbd1552.in‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1285.in‚0$ disk.nvme0n1.out‚604.67041015625È{x:2à - _wandbruntime‚1907  _runtime‚1907N…nÉÄ Ã % _timestamp‚1.7858121625750747e+09 _runtime‚1907.765026147 _step‚356 ' lr-AdamW/pg1‚0.00011594906699739214 ' lr-AdamW/pg2‚0.00011594906699739214 trainer/global_step‚4499“þœ}ÎÅ È ' lr-AdamW/pg2‚0.00011594906699739214 trainer/global_step‚4499 % _timestamp‚1.7858121625750747e+09 _runtime‚1907.765026147 _step‚356 ' lr-AdamW/pg1‚0.00011594906699739214äTâÓå2Æ - _wandbruntime‚1907  _runtime‚1907C–°¾'Ç ¸O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚77437849934848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 % _timestamp‚1.7858121625760481e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29898077 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17718895 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚77437874838528 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚81214207716864 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3949558592140 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚4499 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚75335337156832 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47526485 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚81214312185856 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚79284878709612 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚79284981552096 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3959248540160 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30602272 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4066406011392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚77254959176704 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3959335969792 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12883375 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3949644395264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29898077 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29898079 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47526485 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚81504280849920 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚81504125272064 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚75335320117472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12883119 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚81214312185856 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17718897 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77414906 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚77254976216064 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3959335969792 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚81214207716864 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47516829 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30602014 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3959248540160 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚77254976216064 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚77424564 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4066275337216 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77414906 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚77254976216064 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3959335969792 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚81214207716864 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47516829 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30602014 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3959248540160 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚77254976216064 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚77424564 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4066275337216 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚81214207716864 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3949558592140 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚4499 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚75335337156832 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47526485 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚81214312185856 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚79284878709612 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚79284981552096 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3959248540160 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30602272 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4066406011392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚77254959176704 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3959335969792 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12883375 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3949644395264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29898077å#~²2É - _wandbruntime‚1908  _runtime‚19089›pÕÊ ÁM TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30609013 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77441744 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29904720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3960128159744 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3960128159744 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚75352077329152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚79302496355154 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 $ _timestamp‚1.785812162930825e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47537022 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚79302599197750 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚1908.121106125 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚81232358575616 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77432072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚77455057051648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚77272142979072 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3950436065362 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚4499 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚81522385256448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚234 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚77272125939712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29904722 LDDeviceStatsMonitor.on_train_batch_end/inactive_splÙËò it.small_pool.peak‚538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚81232358575616 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 _step‚358 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77432072 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47527352 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12885945 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4067303301120 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚77272142979072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3960215596544 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3950521868598 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3960215596544 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47527352 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30608777 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118084096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17722832 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚77272125939712·þl<Ì&Ë ÆM NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47527352 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30608777 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118084096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17722832 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚77272125939712 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30609013 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77441744 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29904720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3960128159744 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3960128159744 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚75352077329152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚79302496355154 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 $ _timestamp‚1.785812162930825e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47537022 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚79302599197750 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚1908.121106125 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚81232358575616 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77432072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚77455057051648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚77272142979072 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3950436065362 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚4499 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚81522385256448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚234 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚77272125939712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29904722 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚81232358575616 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 _step‚358 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77432072 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47527352 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12885945 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4067303301120 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚77272142979072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3960215596544 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3950521868598 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3960215596544æºÕm2Ì - _wandbruntime‚1908  _runtime‚1908žé~ûÍ ý 1loss/SRqqt2/detection_loss‚1.9982104086033676 .loss/FBt2/detection_loss‚2.063281840857597 trainer/global_step‚4499 _runtime‚1908.121816655 /loss/FRt1/detection_loss‚2.5635680956176565 .loss/FBt1/detection_loss‚2.083798090616862 _step‚359 1loss/FBt2/assignment_loss‚0.10270853140661654 /loss/FRt2/detection_loss‚2.4850309909809205 1loss/SRqqt1/detection_loss‚0.8641045659012695 /loss/FRt1/assignment_loss‚4.137844108328035 2loss/SRqqt2/assignment_loss‚1.5338049990975338 epoch‚0 $ _timestamp‚1.785812162932023e+09 &loss/total_loss‚23.432273452811014 0loss/FRt2/assignment_loss‚3.5475436590966725 2loss/SRqqt1/assignment_loss‚1.3346255785636318 0loss/FBt1/assignment_loss‚0.7177525837408515ÉÁ¸ùÎ ‚ 1loss/FBt2/assignment_loss‚0.10270853140661654 /loss/FRt2/detection_loss‚2.4850309909809205 1loss/SRqqt1/detection_loss‚0.8641045659012695 /loss/FRt1/assignment_loss‚4.137844108328035 2loss/SRqqt2/assignment_loss‚1.5338049990975338 epoch‚0 $ _timestamp‚1.785812162932023e+09 &loss/total_loss‚23.432273452811014 0loss/FRt2/assignment_loss‚3.5475436590966725 2loss/SRqqt1/assignment_loss‚1.3346255785636318 0loss/FBt1/assignment_loss‚0.7177525837408515 1loss/SRqqt2/detection_loss‚1.9982104086033676 .loss/FBt2/detection_loss‚2.063281840857597 trainer/global_step‚4499 _runtime‚1908.121816655 /loss/FRt1/detection_loss‚2.5635680956176565 .loss/FBt1/detection_loss‚2.083798090616862 _step‚359çñ×뵑Ï ‚(:† ΩÅÓ«Õâ‚ gpu.0.gpu‚29.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚124.482' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚41.494 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦­bÐ ‚(:×  ΩÅÓ™¢«ƒ disk.drbd1965.out‚0$ disk./.usageGB‚586.3506965637207 disk.drbd2041.out‚0 disk.drbd1828.out‚0 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0" proc.memory.rssMB‚ 11254.140625 proc.cpu.threads‚29 network.sent‚ 4.178615e+06 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd2111.out‚0 disk.drbd1552.out‚0 disk.sda2.out‚ 64.5546875 disk.drbd2078.in‚0 disk.drbd2118.in‚0 disk.sda1.out‚0 disk.sda2.in‚ 0.4140625 disk.drbd1931.in‚0 disk.sda.out‚ 64.5546875 disk.rbd0.out‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1584.in‚0 disk.drbd1289.in‚0$ disk.nvme0n1.out‚605.08935546875* proc.memory.availableMB‚38514.68359375 disk.drbd1584.out‚0 disk.drbd1550.out‚0! disk.dm-0.out‚605.08935546875$ memory_percent‚41.23125672340393 disk.drbd1931.out‚0 disk.drbd1285.in‚0 disk.dm-0.in‚2.46875 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.drbd1817.in‚0 network.recv‚312829 disk.drbd2078.out‚0 disk.drbd1673.in‚0 disk.rbd1.out‚ 7.8818359375 cpu‚13.826973876337593 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0) disk./.usagePercent‚32.79180046931594 disk.drbd1356.out‚0 disk.sda.in‚ 0.4140625 disk.drbd1289.out‚0) proc.memory.percent‚17.17245578765869ñÔ]2Ñ - _wandbruntime‚1928  _runtime‚1928R1ÈÒ Â _step‚360 ' lr-AdamW/pg1‚0.00011723767632165744 ' lr-AdamW/pg2‚0.00011723767632165744 trainer/global_step‚4549 % _timestamp‚1.7858121836738765e+09 _runtime‚ 1928.86381468åXÍÍÓ Ç ' lr-AdamW/pg2‚0.00011723767632165744 trainer/global_step‚4549 % _timestamp‚1.7858121836738765e+09 _runtime‚ 1928.86381468 _step‚360 ' lr-AdamW/pg1‚0.00011723767632165744èh‚2Ô - _wandbruntime‚1928  _runtime‚1928}6¾'Õ ¸O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4003244675584 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48044831 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚78298205774848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚80165775789120 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚78113314366464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13026820 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚1928.868632919 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚78113297327104 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4003244675584 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚82409825984000 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78284716 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30230229 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30230227 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚82116542002688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4111595305472 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48054487 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚82116542002688 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4111458339840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 % _timestamp‚1.7858121836747878e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17915745 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4003332105216 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30230227 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚78298230678528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚78113297327104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30230229 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30942565 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚4549 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚82116646471680 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4003332105216 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚82116646471680 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48054487 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚76172328733472 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13027100 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚76172345772832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _step‚361GsÉÖ ½O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚78113297327104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30230229 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30942565 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚4549 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚82116646471680 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4003332105216 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚82116646471680 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48054487 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚76172328733472 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13027100 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚76172345772832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 _step‚361 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4003244675584 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48044831 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚78298205774848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚80165775789120 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚78113314366464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13026820 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚1928.868632919 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚78113297327104 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4003244675584 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚82409825984000 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78284716 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30230229 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30230227 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚82116542002688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4111595305472 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48054487 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚82116542002688 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4111458339840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 % _timestamp‚1.7858121836747878e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17915745 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4003332105216 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30230227 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚78298230678528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30949422 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48055354 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30236872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17919682 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13029992 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48055354 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚82427778539008 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4004211731968 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4112494217216 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚76189085945152 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚78315437795328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚82134692861440 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30236872 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚82427932012544 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚78315412891648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858121840228312e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30236870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚80183496277258 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48065024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚78130464090112 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4112365647360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚82134588385280 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4004124295168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚80183393434662 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78301896 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚78130481129472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3994410332106 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4004124295168 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚78292224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78301896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚78130464090112 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4112365647360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚82134588385280 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4004124295168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚80183393434662 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78301896 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚78130481129472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3994410332106 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4004124295168 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚78292224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78301896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30949422 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48055354 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30236872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17919682 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13029992 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48055354 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚82427778539008 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4004211731968 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4112494217216 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚76189085945152 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚78315437795328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚82134692861440 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30236872 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚82427932012544 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚78315412891648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858121840228312e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30236870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚80183496277258 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48065024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0ê'àʤ2Ú - _wandbruntime‚1929  _runtime‚1929˜ ’gÛ ý 0loss/FRt1/assignment_loss‚3.9199139912923178 0loss/FRt2/assignment_loss‚3.7673405857829305 1loss/FBt2/assignment_loss‚0.07373291097335748 0loss/SRqqt1/detection_loss‚1.170851098985147 epoch‚0 % _timestamp‚1.7858121840240352e+09 /loss/FRt1/detection_loss‚2.6770595566431683 /loss/FBt2/detection_loss‚3.2576808763874903 _step‚363 2loss/SRqqt2/assignment_loss‚1.2848161948578698 0loss/FBt1/assignment_loss‚0.4991104426847484 1loss/SRqqt2/detection_loss‚2.5934681040900096 %loss/total_loss‚25.15813317007001 _runtime‚1929.213790225 1loss/SRqqt1/assignment_loss‚1.482950711817439 /loss/FRt2/detection_loss‚2.5648189666983368 .loss/FBt1/detection_loss‚1.866389729857194 trainer/global_step‚4549OîDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31282991 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚83315362297344 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4156775778816 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30562379 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4047328240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13170394 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚78971635477504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚79158586518528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚4599 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚78971652516864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48572833 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_starw)>Ýt/inactive_split.large_pool.freed‚18112595 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚1949.538642763 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30562379 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4156645104640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚365 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚83018876288512 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚267 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18112597 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48582489 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚83018876288512 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4047240811008 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48582489 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚77009337349472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4047240811008 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚83018980757504 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚83315206719488 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚79158561614848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31282724 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚81046672868628 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚79135210 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚78971652516864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚77009354388832 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚81046775711112 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚78971635477504 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4037421322280 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4037335519156 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858122043454485e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚83018980757504 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚78971652516864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚77009354388832 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚81046775711112 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚78971635477504 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4037421322280 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4037335519156 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858122043454485e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚83018980757504 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31282991 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚83315362297344 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4156775778816 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30562379 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4047328240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13170394 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚78971635477504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚79158586518528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚4599 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚78971652516864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48572833 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18112595 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚1949.538642763 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30562379 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4156645104640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚365 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚83018876288512 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚267 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18112597 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48582489 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚83018876288512 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4047240811008 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48582489 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚77009337349472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4047240811008 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121íDŒ‘Í2å - _wandbruntime‚1949  _runtime‚1949})ÜéÈ&æ ÂM ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4157547193856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79162048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30569020 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13173144 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚83333315925504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79162048 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚81064393356766 _runtime‚1949.888640319 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4048207867392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13172899 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚81064290514170 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79152376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31289431 _step‚366 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48593026 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚78988802240512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4157684152320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚77026094561152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚78988819279872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18116534 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4038212992378 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4048120430592 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30569022 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888”è´žCç ÇM ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚83333477787648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79152376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31289431 _step‚366 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48593026 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚78988802240512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4157684152320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚77026094561152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚78988819279872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18116534 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4038212992378 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4048120430592 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30569022 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4157547193856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79162048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30569020 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13173144 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚83333315925504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79162048 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚81064393356766 _runtime‚1949.888640319 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4048207867392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13172899 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚81064290514170 ODeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚83921210574336 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4201825926656 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13312791 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49110491 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31622236 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚83921315043328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚81927672790620 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚4649 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚79995362 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30894527 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4081223982664 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80005020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49100835 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚79829990667264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4201960795136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4091236946432 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚79829973627904 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚84220903153664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858122252244518e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚79829990667264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30894529 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30894527 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚77846345965472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30894529 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚77846363004832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640ߥ cÂ'ô ¼O QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31622236 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚83921315043328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚81927672790620 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚4649 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚79995362 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30894527 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4081223982664 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80005020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49100835 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚79829990667264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4201960795136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4091236946432 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚79829973627904 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚84220903153664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858122252244518e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚79829990667264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30894529 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30894527 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚77846345965472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30894529 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚77846363004832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18309445 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚83921210574336 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4201825926656 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13312791 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49110491 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680ñIÔ}„2õ - _wandbruntime‚1970  _runtime‚1970ǤM¿Bö ÂM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4092116566016 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49121028 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_bat·Íˆch_end/inactive_split_bytes.all.peak‚ 187192320 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚81945290436274 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚77863086137792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30901172 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31629159 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚83939361433088 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4092204002816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30901170 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80022200 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49111358 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4202874624512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4082101455886 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80012528 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18313384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858122255852072e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4092116566016 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚267 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚79847140390912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚83939256956928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49121028 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18313382 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚84238866432000 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31628890 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚80036124571648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4202741860352 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚77863103177152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13315508 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚83939361433088 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160)Ûx§Í&÷ ÇM R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚83939256956928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49121028 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18313382 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚84238866432000 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31628890 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚80036124571648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4202741860352 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚77863103177152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13315508 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚83939361433088 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4092116566016 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49121028 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚81945290436274 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚77863086137792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30901172 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31629159 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚83939361433088 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4092204002816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30901170 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80022200 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49111358 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4202874624512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4082101455886 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80012528 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18313384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858122255852072e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4092116566016 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚267 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚79847140390912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0òË 1ú2ø - _wandbruntime‚1970  _runtime‚1970Èóiù ü _runtime‚1970.776230278 _step‚371 0loss/FRt1/assignment_loss‚3.7039408139240595 0loss/FRt2/assignment_loss‚3.7141273653650857 1loss/FBt2/assignment_loss‚0.03494748790506848 .loss/FRt1/detection_loss‚2.422210723529627 1loss/SRqqt1/detection_loss‚0.9292876165981412 .loss/FBt2/detection_loss‚1.722948462855489 # _timestamp‚1.78581222558644e+09 2loss/SRqqt1/assignment_loss‚1.4852502978727466 /loss/FRt2/detection_loss‚2.5039075997220466 1loss/SRqqt2/detection_loss‚2.0191149496856857 trainer/global_step‚4649 2loss/SRqqt2/assignment_loss‚1.5133080265539534 0loss/FBt1/assignment_loss‚0.5778475016355514 .loss/FBt1/detection_loss‚1.195384407043459 &loss/total_loss‚21.822275252690915 epoch‚0Ú1!ú  2loss/SRqqt2/assignment_loss‚1.5133080265539534 0loss/FBt1/assignment_loss‚0.5778475016355514 .loss/FBt1/detection_loss‚1.195384407043459 &loss/total_loss‚21.822275252690915 epoch‚0 _runtime‚1970.776230278 _step‚371 0loss/FRt1/assignment_loss‚3.7039408139240595 0loss/FRt2/assignment_loss‚3.7141273653650857 1loss/FBt2/assignment_loss‚0.03494748790506848 .loss/FRt1/detection_loss‚2.422210723529627 1loss/SRqqt1/detection_loss‚0.9292876165981412 .loss/FBt2/detection_loss‚1.722948462855489 # _timestamp‚1.78581222558644e+09 2loss/SRqqt1/assignment_loss‚1.4852502978727466 /loss/FRt2/detection_loss‚2.5039075997220466 1loss/SRqqt2/detection_loss‚2.0191149496856857 trainer/global_step‚4649óu8¿3‘û ‚(:† ŠªÅÓÜ×¢ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚107.097' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚35.699 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0cÜ6nXü ‚(:Í  ŠªÅÓ—Ń disk.drbd1938.in‚0$ memory_percent‚41.23116731643677" proc.memory.rssMB‚ 11254.140625 disk.drbd1356.out‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 cpu‚13.822140876872227 proc.cpu.threads‚29 disk.drbd1931.in‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.drbd2118.in‚0) proc.memory.availableMB‚ 38514.7421875# disk./.usageGB‚586.350700378418 disk.sda2.in‚ 0.4140625 disk.sda2.out‚ 66.56640625 disk.drbd1931.out‚0 disk.drbd1965.out‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 network.recv‚321121 disk.drbd1584.in‚0 disk.drbd1550.out‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0) disk./.usagePercent‚32.79180068265378 disk.nvme0n1.in‚2.46875 disk.sda.out‚ 66.56640625 disk.drbd1226.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.dm-0.out‚ 606.12109375 disk.drbd1817.in‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd1828.in‚0 disk.drbd1552.out‚0 network.sent‚ 4.31655e+06 disk.drbd1356.in‚0 disk.drbd1584.out‚0 disk.drbd2078.out‚0 disk.drbd2041.in‚0 disk.drbd1285.in‚0 disk.drbd1289.in‚0 disk.rbd1.in‚0! disk.nvme0n1.out‚ 606.12109375 disk.drbd1510.out‚0 disk.drbd1938.out‚0) proc.memory.percent‚17.17245578765869 disk.rbd0.out‚0 disk.dm-0.in‚2.46875 disk.drbd2111.in‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.rbd1.out‚ 8.0703125 disk.drbd2041.out‚0 disk.sda.in‚ 0.4140625 disk.drbd1226.out‚0 disk.drbd2118.out‚0¹“¶z2ý - _wandbruntime‚1991  _runtime‚1991ã¯&ºÈþ  $ _timestamp‚1.785812246325207e+09 _runtime‚1991.515144006 _step‚372 ' lr-AdamW/pg1‚0.00012110350429445335 ' lr-AdamW/pg2‚0.00012110350429445335 trainer/global_step‚46992ÅŽÐÍÿ Ç trainer/global_step‚4699 $ _timestamp‚1.785812246325207e+09 _runtime‚1991.515144006 _step‚372 ' lr-AdamW/pg1‚0.00012110350429445335 ' lr-AdamW/pg2‚0.00012110350429445335ô’–D2€ - _wandbruntime‚1991  _runtime‚1991UøÎ|¾' ¸O UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚84823649329152 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚80879298198528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 _runtime‚1991.519883614 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚84823544860160 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18506297 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31226679 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚80688328817664 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49628837 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚82808467027644 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚78683371620832 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚80688311778304 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49638493 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31962490 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚80879273294848 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31226677 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31226679 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚85126489163264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4247053999104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4125198249296 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4247190964736 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚78683354581472 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚84823544860160 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13455933 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚80688328817664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4135233081856 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _step‚373 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚82808569870128 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚84823649329152 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4135320511488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31962228 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚85126327293952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4699 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80855514 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4135233081856 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49628837 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7858122463261135e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31226677 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18506297 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31226679 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚80688328817664 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49628837 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚82808467027644 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚78683371620832 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚80688311778304 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49638493 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31962490 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚80879273294848 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31226677 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31226679 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚85126489163264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4247053999104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4125198249296 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4247190964736 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚78683354581472 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚84823544860160 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13455933 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚80688328817664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4135233081856 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 _step‚373 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚82808569870128 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚84823649329152 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4135320511488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31962228 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚85126327293952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4699 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80855514 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4135233081856 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49628837 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7858122463261135e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31226677 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49639360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4136200138240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚78700094753792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4136112701440 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4248092039168 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1991.872609728 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚82826187515782 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚80705495580672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31233320 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80882352 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚80872680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31233322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49639360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚84841695718912 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚80705478541312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4126075722630 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80872680 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49649030 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚78700111793152 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31969032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚80896480411648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0hZT"Í&… ÇM REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49649030 trainer/global_step‚4699 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4136112701440 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4248092039168 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1991.872609728 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚82826187515782 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚80705495580672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31233320 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80882352 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚80872680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31233322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49639360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚84841695718912 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚80705478541312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4126075722630 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80872680 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49649030 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚78700111793152 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31969032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚80896480411648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49639360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4136200138240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚78700094753792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EHÂ2 - _wandbruntime‚2012  _runtime‚2012?a› ¾'‘ ¸O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚81715666 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18703147 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚81739629134848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4179229217280 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚85725879145984 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18703145 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚86032019065344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚83689364107152 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4292228061184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4169000909680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 _runtime‚2012.395628731 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31558829 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚83689466949636 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31558827 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79520380236832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50166495 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚85725983614976 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚81739654038528 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32302221 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚79520363197472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31558827 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4292365026816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4169086712804 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85725879145984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4179316646912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚4749 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81725324 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4179229217280 _step‚377 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50156839 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85725879145984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4179316646912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚4749 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81725324 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4179229217280 _step‚377 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50156839 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31558829 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚83689466949636 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31558827 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79520380236832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50166495 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚85725983614976 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚81739654038528 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32302221 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚79520363197472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31558827ùIш2“ - _wandbruntime‚2012  _runtime‚2012ÚÔoý$” ÂM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚81742504 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31565470 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31565472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚81756836251648 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13601934 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 % _timestamp‚1.7858122675561738e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚85743925528576 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚81563833731072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81732832 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚86049966975488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚4749 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚81756861155328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4169878382902 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚2012.746440565 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50177032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚237 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13601697 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86050126740480 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32308779 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4169964186138 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4180108836864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81742504 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚81563816691712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50177032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚83707084595290 _step‚378 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18707084 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚81732832 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚81563833731072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚83706981752694 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4180196273664 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚79537103369792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚81563833731072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81732832 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚86049966975488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚4749 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚81756861155328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4169878382902 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚2012.746440565 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50177032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚237 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13601697 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86050126740480 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32308779 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4169964186138 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4180108836864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81742504 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚81563816691712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50177032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚83707084595290 _step‚378 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18707084 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚81732832 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚81563833731072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚83706981752694 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4180196273664 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚79537103369792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31565470 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31565472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992úÐzW2– - _wandbruntime‚2012  _runtime‚2012[6,— ý 0loss/FRt2/assignment_loss‚3.3565918861365898 % _timestamp‚1.7858122675573645e+09 _runtime‚2012.747151656 /loss/FRt1/assignment_loss‚3.918476858933767 2loss/SRqqt1/assignment_loss‚1.5151629298199694 1loss/FBt2/assignment_loss‚0.11136142811518755 /loss/FBt2/detection_loss‚1.6284956104965593 _step‚379 .loss/FRt1/detection_loss‚2.503878721793493 .loss/FRt2/detection_loss‚2.624717060385681 0loss/SRqqt1/detection_loss‚0.991577971256081 &loss/total_loss‚23.360849172020632 epoch‚0 2loss/SRqqt2/assignment_loss‚1.4569941887898104 0loss/FBt1/assignment_loss‚0.5420496300688352 1loss/SRqqt2/detection_loss‚2.2736588656902312 /loss/FBt1/detection_loss‚2.4378840205344288 trainer/global_step‚4749¦©ó˜ ‚ 0loss/FBt1/assignment_loss‚0.5420496300688352 1loss/SRqqt2/detection_loss‚2.2736588656902312 /loss/FBt1/detection_loss‚2.4378840205344288 trainer/global_step‚4749 0loss/FRt2/assignment_loss‚3.3565918861365898 % _timestamp‚1.7858122675573645e+09 _runtime‚2012.747151656 /loss/FRt1/assignment_loss‚3.918476858933767 2loss/SRqqt1/assignment_loss‚1.5151629298199694 1loss/FBt2/assignment_loss‚0.11136142811518755 /loss/FBt2/detection_loss‚1.6284956104965593 _step‚379 .loss/FRt1/detection_loss‚2.503878721793493 .loss/FRt2/detection_loss‚2.624717060385681 0loss/SRqqt1/detection_loss‚0.991577971256081 &loss/total_loss‚23.360849172020632 epoch‚0 2loss/SRqqt2/assignment_loss‚1.4569941887898104ûö|K)œ™ ‚(:‘ ·ªÅÓÖ솃 gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚128.453' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.81766666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0·#»fš ‚(:Û  ·ªÅÓ„Ѓ disk.drbd1673.out‚0 disk.drbd1931.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚2.46875) disk./.usagePercent‚32.79180089599162 disk.sda1.in‚0 network.recv‚326819 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0$ proc.memory.rssMB‚11253.87109375 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.drbd1817.out‚0 disk.drbd1931.in‚0 disk.drbd1828.in‚0 disk.drbd2041.in‚0 disk.sda.out‚ 67.73828125 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.rbd0.out‚0 disk.drbd1965.in‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.4140625 disk.drbd1584.in‚0 disk.drbd1226.in‚0 cpu‚13.818908695284254 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.dm-0.in‚2.46875 disk.drbd1289.in‚0 network.sent‚ 4.409569e+06 disk.drbd1938.out‚0 disk.rbd1.out‚ 8.1962890625$ disk.nvme0n1.out‚606.73779296875 disk.drbd1356.in‚0 disk.drbd2111.in‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.drbd2078.out‚0 disk.sda.in‚ 0.4140625 disk.drbd1550.in‚0! disk.dm-0.out‚606.73779296875 disk.drbd1510.in‚0* proc.memory.availableMB‚38513.95703125$ disk./.usageGB‚586.3507041931152 disk.drbd1673.in‚0 disk.drbd1584.out‚0$ memory_percent‚41.23236536979675 proc.cpu.threads‚29 disk.sda2.out‚ 67.73828125 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0) proc.memory.percent‚17.17204451560974¼L¤í2› - _wandbruntime‚2033  _runtime‚2033%Ÿ}Èœ  _runtime‚2033.206986302 _step‚380 ' lr-AdamW/pg1‚0.00012368072294298397 ' lr-AdamW/pg2‚0.00012368072294298397 trainer/global_step‚4799 $ _timestamp‚1.785812288017047e+09Ÿ9âNÍ Ç ' lr-AdamW/pg1‚0.00012368072294298397 ' lr-AdamW/pg2‚0.00012368072294298397 trainer/global_step‚4799 $ _timestamp‚1.785812288017047e+09 _runtime‚2033.206986302 _step‚380ü Õ22ž - _wandbruntime‚2033  _runtime‚2033ñ‚úC¾'Ÿ ¸O 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚86628213431808 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4337530009088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31890977 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4223312782336 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32643448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚80357371813472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4223225352704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚82404988079104 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4223312782336 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82585476 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚86937384309760 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50694497 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚86628213431808 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50684841 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚86937539887616 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚86628317900800 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚82404988079104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31890977 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4223312782336 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32643448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚80357371813472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4223225352704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚82404988079104 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4223312782336 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82585476 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚86937384309760 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50694497 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚86628213431808 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50684841 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚86937539887616 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚86628317900800 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚82404988079104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4338435163136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚4799 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4224192409088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚86646364290560 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31897620 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31897622 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31897620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚82617216995328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚382 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚80374111985792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858122883723142e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18903932 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚80374129025152 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50705034 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50695364 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚82422154842112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50695364 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚82617192091648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82592984 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13746526 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚2033.562601287 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚82422171881472 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82602656 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32650216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚84587981674798 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18903934 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚86646364290560 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4224104972288 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚82422171881472 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚84587878832202 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82602656 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86955652158464 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4338298204672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚242 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538dÓ%®Í&£ ÇM ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚82422171881472 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82602656 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32650216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚84587981674798 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18903934 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚86646364290560 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4224104972288 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚82422171881472 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚84587878832202 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82602656 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86955652158464 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4338298204672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚242 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50705034 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚82422154842112 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4224192409088 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13746284 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4338435163136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚4799 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4224192409088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚86646364290560 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31897620 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31897622 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31897620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚82617216995328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚382 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚80374111985792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858122883723142e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18903932 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚80374129025152 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50705034 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50695364 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚82422154842112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50695364 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚82617192091648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82592984 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13746526 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚2033.562601287 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0þŸ‚˜$2¤ - _wandbruntime‚2033  _runtime‚2033p£ï£¥ þ /loss/FRt2/assignment_loss‚3.540158480740665 1loss/SRqqt2/detection_loss‚1.9546912432794112 /loss/FBt2/detection_loss‚1.3775237093952586 _step‚383 2loss/SRqqt1/assignment_loss‚1.4410362967186503 2loss/SRqqt2/assignment_loss‚1.4343747294631348 1loss/FBt2/assignment_loss‚0.05812252248929132 .loss/FRt2/detection_loss‚2.545653893706504 /loss/FBt1/detection_loss‚1.5959215117269197 epoch‚0 % _timestamp‚1.7858122883735137e+09 _runtime‚2033.563419379 0loss/FRt1/assignment_loss‚4.0204761175765205 /loss/FRt1/detection_loss‚2.3557833741927885 1loss/SRqqt1/detection_loss‚0.9550604394487164 0loss/FBt1/assignment_loss‚0.6100062452256679 %loss/total_loss‚21.88880856396353 trainer/global_step‚4799Ô9%• ¦ ƒ /loss/FRt2/assignment_loss‚3.540158480740665 1loss/SRqqt2/detection_loss‚1.9546912432794112 /loss/FBt2/detection_loss‚1.3775237093952586 _step‚383 2loss/SRqqt1/assignment_loss‚1.4410362967186503 2loss/SRqqt2/assignment_loss‚1.4343747294631348 1loss/FBt2/assignment_loss‚0.05812252248929132 .loss/FRt2/detection_loss‚2.545653893706504 /loss/FBt1/detection_loss‚1.5959215117269197 epoch‚0 % _timestamp‚1.7858122883735137e+09 _runtime‚2033.563419379 0loss/FRt1/assignment_loss‚4.0204761175765205 /loss/FRt1/detection_loss‚2.3557833741927885 1loss/SRqqt1/detection_loss‚0.9550604394487164 0loss/FBt1/assignment_loss‚0.6100062452256679 %loss/total_loss‚21.88880856396353 trainer/global_step‚4799ÿÔ¸Gœ§ ‚(:‘ ƪÅÓÌ‘Õƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚100.948' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚33.64933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÈÍ’êe¨ ‚(:Ú  ƪÅÓçöèƒ disk.drbd1584.out‚0 network.recv‚329383 disk.drbd1550.in‚0 disk.drbd1285.out‚0 disk.drbd1285.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 8.2587890625 disk.drbd2078.out‚0 cpu‚13.81782015888481 disk.drbd2111.out‚0 disk.drbd2118.out‚0$ proc.memory.rssMB‚11253.87109375 disk.drbd1510.out‚0 disk.drbd1226.out‚0 disk.drbd1828.out‚0 disk.sda2.in‚ 0.4140625$ disk.nvme0n1.out‚607.08154296875 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.drbd1289.out‚0 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚2.46875 disk.drbd1673.in‚0 disk.drbd1965.in‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.sda2.out‚ 68.08203125 disk.drbd1356.out‚0) disk./.usagePercent‚32.79180089599162 disk.drbd2111.in‚0) proc.memory.percent‚17.17204451560974$ disk./.usageGB‚586.3507041931152 disk.drbd1938.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 68.08203125! disk.dm-0.out‚607.08154296875 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1356.in‚0 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.4140625 proc.cpu.threads‚29 disk.drbd1550.out‚0 disk.drbd1289.in‚0 disk.drbd1931.out‚0$ memory_percent‚41.23308062553406* proc.memory.availableMB‚38513.48828125 network.sent‚ 4.454577e+06 disk.drbd1828.in‚0]Ô³ú2© - _wandbruntime‚2053  _runtime‚2053¿e“ɪ à ' lr-AdamW/pg2‚0.00012496933226724928 trainer/global_step‚4849 % _timestamp‚1.7858123087672436e+09 _runtime‚2053.957189922 _step‚384 ' lr-AdamW/pg1‚0.00012496933226724928üã‚Ϋ È _step‚384 ' lr-AdamW/pg1‚0.00012496933226724928 ' lr-AdamW/pg2‚0.00012496933226724928 trainer/global_step‚4849 % _timestamp‚1.7858123087672436e+09 _runtime‚2053.957189922€ÁM%2¬ - _wandbruntime‚2053  _runtime‚2053%»!t­ ¸O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 % _timestamp‚1.7858123087681658e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83445628 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4256863639820 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4267221488128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4267308917760 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚83460340814848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚87530547717632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32223127 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4267221488128 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32223127 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚87843080696320 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚81194397468832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51212843 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32983332 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13886487 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19096847 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32223129 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4267308917760 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚87530652186624 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚83263343268864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51212843 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚87530547717632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32983582 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83435970 \IDe³$K3JviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚83263343268864 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚83460365718528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚85451261108652 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4849 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51222499 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19096845 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4382582206464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13886735 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4256777836696 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚87530652186624 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32223129 _runtime‚2053.961629541 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51222499 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚83263326229504 _step‚385)‘Ï"Ã'® ½O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32223127 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4267221488128 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32223127 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚87843080696320 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚81194397468832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51212843 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32983332 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13886487 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19096847 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32223129 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4267308917760 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚87530652186624 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚83263343268864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51212843 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚87530547717632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32983582 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83435970 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚83263343268864 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚83460365718528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚85451261108652 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4849 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51222499 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19096845 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4382582206464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13886735 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4256777836696 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚87530652186624 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32223129 _runtime‚2053.961629541 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51222499 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚83263326229504 _step‚385 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 % _timestamp‚1.7858123087681658e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83445628 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4256863639820 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4267221488128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4267308917760 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚83460340814848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚87530547717632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328… ¡2¯ - _wandbruntime‚2054  _runtime‚20548·&AÈ&° ÂM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚237 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13889330 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚2054.309543682 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83462808 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32990112 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚235 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19100782 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32229770 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4268188544512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83462808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4383616445440 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4383492069888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚85468775911710 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚83280492992512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4268188544512 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51233036 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32990349 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚81211120601792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19100784 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83453136 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51233036 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚81211137641152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4257655309918 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚87861189280768 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚386 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32229772 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32229772 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚87548594100224 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚83280510031872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚4849 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858123091192508e+09 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 :fDÖ!± ÇM 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚237 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13889330 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚2054.309543682 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83462808 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32990112 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚235 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19100782 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32229770 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4268188544512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83462808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4383616445440 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4383492069888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚85468775911710 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚83280492992512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4268188544512 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51233036 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32990349 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚81211120601792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19100784 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83453136 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51233036 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚81211137641152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4257655309918 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚87861189280768 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚386 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32229772 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32229772 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚87548594100224 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚83280510031872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚4849 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858123091192508e+09‚z!Rö2² - _wandbruntime‚2054  _runtime‚2054ŒçÌo³ ü /loss/FBt1/detection_loss‚1.7246166616678238 _runtime‚2054.310241872 2loss/SRqqt1/assignment_loss‚1.4843007001806707 1loss/SRqqt2/assignment_loss‚1.549940866051298 /loss/FRt1/detection_loss‚2.6633331529472186 &loss/total_loss‚22.144890562394036 epoch‚0 trainer/global_step‚4849 1loss/SRqqt1/detection_loss‚1.1595257606642169 /loss/FBt2/detection_loss‚1.1443051234927288 % _timestamp‚1.7858123091204534e+09 _step‚387 1loss/FBt1/assignment_loss‚0.21977276608959073 ,loss/FRt2/detection_loss‚2.5417709093035 1loss/SRqqt2/detection_loss‚1.9919420985335654 .loss/FRt1/assignment_loss‚3.90575681389242 /loss/FRt2/assignment_loss‚3.755168749962324 2loss/FBt2/assignment_loss‚0.004456959608677842¾RÃ5´  .loss/FRt1/assignment_loss‚3.90575681389242 /loss/FRt2/assignment_loss‚3.755168749962324 2loss/FBt2/assignment_loss‚0.004456959608677842 /loss/FBt1/detection_loss‚1.7246166616678238 _runtime‚2054.310241872 2loss/SRqqt1/assignment_loss‚1.4843007001806707 1loss/SRqqt2/assignment_loss‚1.549940866051298 /loss/FRt1/detection_loss‚2.6633331529472186 &loss/total_loss‚22.144890562394036 epoch‚0 trainer/global_step‚4849 1loss/SRqqt1/detection_loss‚1.1595257606642169 /loss/FBt2/detection_loss‚1.1443051234927288 % _timestamp‚1.7858123091204534e+09 _step‚387 1loss/FBt1/assignment_loss‚0.21977276608959073 ,loss/FRt2/detection_loss‚2.5417709093035 1loss/SRqqt2/detection_loss‚1.9919420985335654ƒ´ÒÃdµ ‚(:Ù  ÕªÅÓ¯Ñïƒ disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.sda.in‚ 0.4140625 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0$ proc.memory.rssMB‚11253.87109375 disk.drbd1673.in‚0 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.drbd2118.in‚0 disk.sda2.out‚ 68.5390625$ memory_percent‚41.23319983482361) proc.memory.percent‚17.17204451560974 cpu‚13.816630949046935 disk.drbd1673.out‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0) disk./.usagePercent‚32.79180089599162 disk.sda.out‚ 68.5390625 disk.rbd1.in‚0 disk.rbd1.out‚ 8.2587890625 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.drbd1938.out‚0 disk.drbd1931.in‚0 network.sent‚ 4.499735e+06 disk.drbd1285.in‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚2.46875* proc.memory.availableMB‚38513.41015625 disk.drbd1550.in‚0 proc.cpu.threads‚29 disk.rbd0.in‚0 disk.drbd2111.out‚0 disk.drbd1552.out‚0 disk.sda2.in‚ 0.4140625 disk.drbd1356.out‚0 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.drbd1550.out‚0! disk.dm-0.out‚607.21826171875 disk.drbd1817.in‚0 disk.drbd1356.in‚0 disk.drbd1552.in‚0$ disk.nvme0n1.out‚607.21826171875 disk.drbd1584.out‚0 disk.drbd1226.in‚0 network.recv‚331810$ disk./.usageGB‚586.3507041931152 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.dm-0.in‚2.46875 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd1817.out‚0 disk.sda1.in‚0)ÅK‘¶ ‚(:† ÕªÅÓϪùƒ gpu.0.gpu‚30.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚128.157' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚42.719 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÖÏ´Qœ· ‚(:‘ äªÅÓ×Ìü‚ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚106.708' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚35.56933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“˜êJ`¸ ‚(:Õ  äªÅÓÇ߃ disk.drbd1510.in‚0 disk.rbd1.out‚ 8.291015625$ disk./.usageGB‚586.3507232666016 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1584.in‚0 disk.sda2.out‚ 68.88671875 disk.rbd1.in‚0 disk.sda.out‚ 68.88671875 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.drbd2111.in‚0 disk.drbd1931.out‚0' proc.memory.availableMB‚ 38513.40625 disk.drbd2041.out‚0 disk.drbd1285.out‚0 disk.drbd1289.in‚0 network.sent‚ 4.54468e+06 disk.drbd1510.out‚0 proc.cpu.threads‚29 disk.sda.in‚ 0.4140625 disk.dm-0.out‚607.5322265625 disk.sda2.in‚ 0.4140625 cpu‚13.815463973037065 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.drbd2111.out‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0 disk.drbd2078.out‚0 disk.drbd1673.out‚0 disk.drbd1938.in‚0% memory_percent‚41.233205795288086) proc.memory.percent‚17.17204451560974 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚2.46875# disk.nvme0n1.out‚607.5322265625 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0) disk./.usagePercent‚32.79180196268083 network.recv‚334184 disk.rbd0.in‚0 disk.drbd1817.in‚0 disk.drbd1828.in‚0 disk.dm-0.in‚2.46875 disk.drbd1938.out‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11253.87109375?S¾Ù2¹ - _wandbruntime‚2075  _runtime‚2075ƾÐɺ à ' lr-AdamW/pg1‚0.00012625794159151456 ' lr-AdamW/pg2‚0.00012625794159151456 trainer/global_step‚4899 % _timestamp‚1.7858123298226995e+09 _runtime‚2075.014616708 _step‚388}/ Ôλ È ' lr-AdamW/pg1‚0.00012625794159151456 ' lr-AdamW/pg2‚0.00012625794159151456 trainer/global_step‚4899 % _timestamp‚1.7858123298226995e+09 _runtime‚2075.014616708 _step‚388„£Ê2¼ - _wandbruntime‚2075  _runtime‚2075¯ ¶'¾'½ ¸O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚84320696654848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858123298233643e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32555277 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4311305053184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51740845 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19293697 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14030070 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚84320721558528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51750501 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚84121664379904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4311217623552 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51750501 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4311217623552 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚88432986472448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32555279 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚88432986472448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚84121681419264 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚88432882003456 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14030328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚82031389045472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4300666300204 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51740845 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33324025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4300752103328 trainer/global_step‚4899 _step‚389 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4311305053184 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚84305780 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚88748661935616 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32555279 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84121664379904 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19293695 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32555279 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚88432986472448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚84121681419264 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚88432882003456 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14030328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚82031389045472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4300666300204 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51740845 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33324025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4300752103328 trainer/global_step‚4899 _step‚389 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4311305053184 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚84305780 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚88748661935616 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32555279 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84121664379904 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19293695 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858123298233643e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32555277 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4311305053184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51740845 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19293697 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14030070 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚84320721558528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51750501 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚84121664379904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4311217623552 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51750501 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4311217623552 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130…×½ŒL2¿ - _wandbruntime‚2075  _runtime‚20754“ ¿À ÀM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51761038 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 _step‚390 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84138831142912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33330825 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚84337928675328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51751368 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4312097243136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32561920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32561922 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33330572 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 # _timestamp‚1.78581233016619e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84138831142912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4428843155456 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4301629576662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19297634 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4312184679936 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚88451032862208 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚84138848182272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚84313288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 __ §yruntime‚2075.356511212 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚84313288 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84322960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚88766614162944 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚88450928386048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32561922 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14033191 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4301543773426 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14032940 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚84337903771648 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚86349672991218 trainer/global_step‚4899 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84138848182272 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚88766771830784 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚82048146257152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚2514¥˜Ë&Á ÅM M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4312184679936 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚88451032862208 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚84138848182272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚84313288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚2075.356511212 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚84313288 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84322960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚88766614162944 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚88450928386048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32561922 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14033191 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4301543773426 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14032940 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚84337903771648 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚86349672991218 trainer/global_step‚4899 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84138848182272 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚88766771830784 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚82048146257152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51761038 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 _step‚390 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84138831142912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33330825 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚84337928675328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51751368 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4312097243136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32561920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32561922 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33330572 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 # _timestamp‚1.78581233016619e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84138831142912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4428843155456 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4301629576662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19297634†¤PO2 - _wandbruntime‚2075  _runtime‚2075ü‰@Éà ý 0loss/FBt1/assignment_loss‚0.3253661072576506 /loss/FBt1/detection_loss‚1.5762131821491367 trainer/global_step‚4899 2loss/SRqqt1/assignment_loss‚1.7077316545288672 .loss/FRt2/detection_loss‚2.459854677152098 /loss/FBt2/detection_loss‚1.1056665672832078 epoch‚0 2loss/SRqqt2/assignment_loss‚1.5287709502073434 1loss/FBt2/assignment_loss‚0.07688855674879384 /loss/FRt1/detection_loss‚2.5494465506390522 0loss/SRqqt2/detection_loss‚2.072167293383525 %loss/total_loss‚21.54558820290483 % _timestamp‚1.7858123301674106e+09 _step‚391 0loss/FRt1/assignment_loss‚3.6930278210263503 /loss/FRt2/assignment_loss‚3.562794911727477 1loss/SRqqt1/detection_loss‚0.8876599308013294 _runtime‚2075.357195058{º 8Ä ‚ 2loss/SRqqt1/assignment_loss‚1.7077316545288672 .loss/FRt2/detection_loss‚2.459854677152098 /loss/FBt2/detection_loss‚1.1056665672832078 epoch‚0 2loss/SRqqt2/assignment_loss‚1.5287709502073434 1loss/FBt2/assignment_loss‚0.07688855674879384 /loss/FRt1/detection_loss‚2.5494465506390522 0loss/SRqqt2/detection_loss‚2.072167293383525 %loss/total_loss‚21.54558820290483 % _timestamp‚1.7858123301674106e+09 _step‚391 0loss/FRt1/assignment_loss‚3.6930278210263503 /loss/FRt2/assignment_loss‚3.562794911727477 1loss/SRqqt1/detection_loss‚0.8876599308013294 _runtime‚2075.357195058 0loss/FBt1/assignment_loss‚0.3253661072576506 /loss/FBt1/detection_loss‚1.5762131821491367 trainer/global_step‚4899‡f‰ÏKdÅ ‚(:Ù  óªÅÓ‚¾ƒ disk.drbd2118.out‚0 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.sda1.in‚0* proc.memory.availableMB‚38513.37109375 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚2.46875 disk.rbd1.out‚ 8.384765625 disk.drbd1356.out‚0 disk.drbd1965.out‚0 proc.cpu.threads‚29 network.sent‚ 4.547805e+06 network.recv‚335263 disk.drbd1510.in‚0 disk.drbd2041.in‚0 disk.sda.in‚ 0.4140625 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.drbd1584.in‚0) proc.memory.percent‚17.17204451560974 cpu‚13.814483565310178 disk.dm-0.in‚2.46875 disk.sda2.in‚ 0.4140625 disk.drbd1931.out‚0% memory_percent‚41.233259439468384 disk.drbd1552.out‚0) disk./.usagePercent‚32.79180089599162 disk.drbd1828.in‚0 disk.drbd1931.in‚0 disk.sda1.out‚0 disk.drbd1550.in‚0 disk.drbd1510.out‚0 disk.drbd1285.in‚0 disk.sda.out‚ 69.31640625 disk.drbd1965.in‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.sda2.out‚ 69.31640625 disk.drbd1289.in‚0 disk.drbd1356.in‚0 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd2118.in‚0 disk.dm-0.out‚607.7158203125 disk.drbd1289.out‚0 disk.drbd1673.out‚0$ proc.memory.rssMB‚11253.87109375# disk.nvme0n1.out‚607.7158203125$ disk./.usageGB‚586.3507041931152 disk.drbd1552.in‚0c\éÆ ‚(:… óªÅÓ¡ÃÁƒ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚96.414' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.138 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0èúÝ2Ç - _wandbruntime‚2095  _runtime‚2095<‡ù®ÇÈ Á trainer/global_step‚4949 # _timestamp‚1.78581235043484e+09 _runtime‚2095.624795669 _step‚392 ' lr-AdamW/pg1‚0.00012754655091577988 ' lr-AdamW/pg2‚0.000127546550915779886hºÌÉ Æ # _timestamp‚1.78581235043484e+09 _runtime‚2095.624795669 _step‚392 ' lr-AdamW/pg1‚0.00012754655091577988 ' lr-AdamW/pg2‚0.00012754655091577988 trainer/global_step‚4949ˆ„¶£2Ê - _wandbruntime‚2095  _runtime‚2095 â|ž'Ë ¸O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14172770 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32887427 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚89335216289280 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84980002530304 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚82868397661472 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14173023 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4355213758976 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4472954139136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4355213758976 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚85165932 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33663570 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4355301188608 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52268847 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858123504357738e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32887427 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚89654168503296 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4355301188608 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚85181077398528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚87213055267668 trainer/global_step‚4949 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19490545 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚85181052494848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚89335216289280 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32887429 _runtime‚2095.629645231 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52268847 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚84980019569664 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33663315 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548tE‘?í Ì ½O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4355213758976 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚85165932 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33663570 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4355301188608 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52268847 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858123504357738e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32887427 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚89654168503296 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4355301188608 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚85181077398528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚87213055267668 trainer/global_step‚4949 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19490545 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚85181052494848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚89335216289280 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32887429 _runtime‚2095.629645231 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52268847 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚84980019569664 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33663315 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14172770 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32887427 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚89335216289280 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84980002530304 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚82868397661472 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14173023 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4355213758976 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4472954139136‰Ýܹ2Í - _wandbruntime‚2095  _runtime‚2095_™ùOÇ&Î ÁM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4345518040170 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4356093378560 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85173440 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚85198284515328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚266 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4356180815360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4473993235968 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85183112 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚89353367148032 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚89353367148032 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85183112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14175613 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52289040 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚84997186332672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4356093378560 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32894072 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84997186332672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33670361 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4356180815360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14175877 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33670095 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4473854180352 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚394 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32894072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19494482 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52279370 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84997169293312 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚87230672913322 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32894070 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52289040 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚85198259611648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚82885137833792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32894070 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚89672113792000 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52279370 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚ 2095.97778407 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84997169293312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚82885154873152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 % _timestamp‚1.7858123507874942e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800Q³QúÌ&Ï ÆM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32894072 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84997186332672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33670361 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4356180815360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14175877 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33670095 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4473854180352 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 _step‚394 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32894072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19494482 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52279370 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84997169293312 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚87230672913322 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32894070 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52289040 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚85198259611648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚82885137833792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32894070 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚89672113792000 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52279370 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚ 2095.97778407 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84997169293312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚82885154873152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 % _timestamp‚1.7858123507874942e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4345518040170 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4356093378560 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85173440 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚85198284515328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚266 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4356180815360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4473993235968 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85183112 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚89353367148032 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚89353367148032 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85183112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14175613 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52289040 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚84997186332672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4356093378560 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121Š€õ/_2Ð - _wandbruntime‚2095  _runtime‚2095pcñÑ û 2loss/SRqqt1/assignment_loss‚1.5574416348999236 1loss/FBt2/assignment_loss‚0.15861215207244392 1loss/SRqqt2/detection_loss‚1.9358529618808202 _runtime‚ 2095.97859555 .loss/FRt1/detection_loss‚2.327074827419387 1loss/SRqqt1/detection_loss‚1.0344401595265558 &loss/total_loss‚21.966731971360684 _step‚395 epoch‚0 0loss/FRt1/assignment_loss‚3.6760489138556114 /loss/FRt2/assignment_loss‚3.611753389239311 0loss/FBt1/assignment_loss‚0.8173966004720165 /loss/FRt2/detection_loss‚2.6447869025170805 .loss/FBt1/detection_loss‚0.607966632499079 trainer/global_step‚4949 % _timestamp‚1.7858123507886682e+09 1loss/SRqqt2/assignment_loss‚2.011099718191794 .loss/FBt2/detection_loss‚1.584258078786661ƒ(™Ò € .loss/FRt1/detection_loss‚2.327074827419387 1loss/SRqqt1/detection_loss‚1.0344401595265558 &loss/total_loss‚21.966731971360684 _step‚395 epoch‚0 0loss/FRt1/assignment_loss‚3.6760489138556114 /loss/FRt2/assignment_loss‚3.611753389239311 0loss/FBt1/assignment_loss‚0.8173966004720165 /loss/FRt2/detection_loss‚2.6447869025170805 .loss/FBt1/detection_loss‚0.607966632499079 trainer/global_step‚4949 % _timestamp‚1.7858123507886682e+09 1loss/SRqqt2/assignment_loss‚2.011099718191794 .loss/FBt2/detection_loss‚1.584258078786661 2loss/SRqqt1/assignment_loss‚1.5574416348999236 1loss/FBt2/assignment_loss‚0.15861215207244392 1loss/SRqqt2/detection_loss‚1.9358529618808202 _runtime‚ 2095.97859555‹1x‡NÓ ‚(:„ ‚«ÅÓö·»ƒ gpu.0.gpu‚29.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚101.82' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚33.94 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÉOIJ`Ô ‚(:Õ  ‚«ÅÓ§¶Ðƒ disk.drbd1965.out‚0 disk.drbd1289.in‚0 disk.drbd1817.out‚0 disk.drbd2078.out‚0 disk.dm-0.out‚608.0537109375 disk.sda1.out‚0 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd1356.in‚0 proc.cpu.threads‚29 disk.sda.out‚ 69.69921875 disk.drbd1550.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd1550.in‚0 disk.drbd1285.out‚0 cpu‚13.813456579670737 disk.drbd2111.in‚0 disk.drbd1931.in‚0 disk.rbd1.out‚ 8.533203125 disk.nvme0n1.in‚2.46875 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1584.out‚0 disk.dm-0.in‚2.46875 disk.drbd1552.in‚0* proc.memory.percent‚17.172062397003174 network.sent‚ 4.592791e+06 disk.rbd0.in‚0 disk.drbd1510.in‚0$ memory_percent‚41.23358726501465 network.recv‚337753 disk.drbd1938.out‚0# proc.memory.rssMB‚ 11253.8828125) disk./.usagePercent‚32.79180196268083 disk.drbd2078.in‚0 disk.drbd1510.out‚0 disk.drbd2118.out‚0# disk.nvme0n1.out‚608.0537109375 disk.drbd2111.out‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0' proc.memory.availableMB‚ 38513.15625 disk.sda2.in‚ 0.4140625 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0$ disk./.usageGB‚586.3507232666016 disk.drbd1673.out‚0 disk.drbd1584.in‚0 disk.rbd0.out‚0 disk.sda.in‚ 0.4140625 disk.sda1.in‚0 disk.sda2.out‚ 69.69921875 disk.drbd1931.out‚0ëJª›Õ ‚(: ‘«ÅÓºÓ›ƒ gpu.0.gpu‚21.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚95.75' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚31.916666666666664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0àWý=dÖ ‚(:Ù  ‘«ÅÓµµàƒ disk.rbd1.out‚ 8.533203125 disk.drbd1226.in‚0 disk.drbd2118.in‚0 disk.drbd1673.out‚0$ disk.nvme0n1.out‚608.37158203125 disk.sda1.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚2.46875 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.drbd2041.out‚0 disk.drbd1938.out‚0 disk.drbd1965.out‚0 disk.drbd1285.in‚0 disk.drbd1931.in‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.rbd0.in‚0! disk.dm-0.out‚608.37158203125 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.nvme0n1.in‚2.46875 proc.cpu.threads‚29 disk.drbd2118.out‚0 disk.sda2.out‚ 70.00390625 network.sent‚ 4.637731e+06 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.drbd1356.in‚0 cpu‚13.812214444298093 disk.drbd1356.out‚0 disk.sda.out‚ 70.00390625# memory_percent‚41.2339985370636$ disk./.usageGB‚586.3507308959961 disk.drbd1510.in‚0 disk.drbd1817.out‚0 disk.drbd2041.in‚0# proc.memory.rssMB‚ 11253.8828125 disk.drbd1965.in‚0 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.drbd1931.out‚0) disk./.usagePercent‚32.79180238935651 disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.drbd1584.out‚0* proc.memory.availableMB‚38512.88671875 network.recv‚340112 disk.sda2.in‚ 0.4140625 disk.sda.in‚ 0.4140625* proc.memory.percent‚17.172062397003174 disk.sda1.out‚0 disk.drbd1938.in‚0‚¥$ˆ2× - _wandbruntime‚2116  _runtime‚2116Ø‚†Ì±Ø Á & lr-AdamW/pg1‚0.0001288351602400452 & lr-AdamW/pg2‚0.0001288351602400452 trainer/global_step‚4999 % _timestamp‚1.7858123713960218e+09 _runtime‚2116.58à+,Æ5955218 _step‚396(iRÀÌÙ Æ _runtime‚2116.585955218 _step‚396 & lr-AdamW/pg1‚0.0001288351602400452 & lr-AdamW/pg2‚0.0001288351602400452 trainer/global_step‚4999 % _timestamp‚1.7858123713960218e+09ŒQhö2Ú - _wandbruntime‚2116  _runtime‚2116úN˾'Û ¸O G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _runtime‚2116.590622307 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚86016426 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚90237655044096 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚90237550575104 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52806505 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4388443227220 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚90237655044096 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚83705406277472 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52796849 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33219579 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4399297324032 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚90559686859776 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚86041433238528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33219579 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚86041408334848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52806505 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚85838357720064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4399209894400 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858123713969738e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚90559531281920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33219577 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4399209894400 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86026084 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33219577 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34003312 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚85838357720064 _step‚397 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚90237550575104 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52806505 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4388443227220 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚90237655044096 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚83705406277472 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52796849 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33219579 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4399297324032 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚90559686859776 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚86041433238528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33219579 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚86041408334848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52806505 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚85838357720064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4399209894400 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858123713969738e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚90559531281920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33219577 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4399209894400 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86026084 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33219577 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34003312 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚85838357720064 _step‚397 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚90255701433856 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52817042 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚88111569992830 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52807372 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90577798712832 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚90255596957696 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33226222 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88111467150234 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚83722146449792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 trainer/global_step‚4999 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚90255701433856 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33226220 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚398 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52817042 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚83722163489152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4519019301888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19691334 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86043264 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚85855507443712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4400176950784 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚85855507443712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4389406503678 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33226222 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚86058640355328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86033592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34009833 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4519158357504 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86043264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14318501 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4389320700442 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856ð|n9ß ÇM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚85855524483072 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88111467150234 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚83722146449792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 WIDeviceStatsMoniªÜé”tor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 trainer/global_step‚4999 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚90255701433856 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33226220 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚398 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52817042 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚83722163489152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4519019301888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19691334 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86043264 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚85855507443712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4400176950784 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚85855507443712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4389406503678 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33226222 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚86058640355328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86033592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34009833 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4519158357504 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86043264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14318501 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4389320700442 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚90255701433856 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52817042 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚88111569992830 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52807372 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90577798712832 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚90255596957696 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33226222 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚91139884860928 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34343638 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4443293459456 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚5049 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4563296753664 _runtime‚2137.357633169 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚91465222797824 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚86696695870464 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4432417493852 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34343380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33551727 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚88974746584200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4443293459456 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚86696695870464 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53324851 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14459135 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚84542431932832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858123921641777e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚84542414893472 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚86876578 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53334507 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33551727 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33551729 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚86696678831104 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _step‚401 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚86696678831104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚91139989329920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4443206029824 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0wÛrÕÃ'ê ½O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚86696678831104 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _step‚401 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚86696678831104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚91139989329920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4443206029824 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚91139884860928 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34343638 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4443293459456 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚5049 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4563296753664 _runtime‚2137.357633169 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚91465222797824 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚86696695870464 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4432417493852 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34343380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33551727 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚88974746584200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4443293459456 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚86696695870464 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53324851 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14459135 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚84542431932832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858123921641777e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚84542414893472 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚86876578 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53334507 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33551727 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33551729‘útÏ2ë - _wandbruntime‚2137  _runtime‚2137ô¦¥*ì ÂM R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚91157931243520 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33558370 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.l³ØDÈž%arge_pool.freed‚84559155065792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53345044 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚86713845594112 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚86713862633472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚86713845594112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚84559172105152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4444173086208 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88992364229742 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86903416 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4433294967186 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 _runtime‚2137.703116311 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19888182 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86893744 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚86918971291648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34350432 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚91483328777216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚402 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33558372 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86903416 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4433209163950 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33558370 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19888184 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4564332581888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34350195ßJdÍ&í ÇM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53335374 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4564204012032 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86903416 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4433294967186 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 _runtime‚2137.703116311 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19888182 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86893744 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚86918971291648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34350432 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚91483328777216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚402 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33558372 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86903416 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4433209163950 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33558370 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19888184 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4564332581888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34350195 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚91157931243520 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33558370 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚84559155065792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53345044 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚86713845594112 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚86713862633472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚86713845594112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚84559172105152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4444173086208 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88992364229742 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33883879 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚87555034020864 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚5099 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34683546 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14602451 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87736730 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚92370641601536 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87746388 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4487202165248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20081095 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33883877 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚92042219146752 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚87762120014848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚89855643663708 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4476305957360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 % _timestamp‚1.7858124129587867e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53852853 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚87555016981504 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14602708 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚87555034020864 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33883879 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚87555034020864 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚5099 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34683546 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14602451 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87736730 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚92370641601536 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87746388 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4487202165248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20081095 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33883877 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚92042219146752 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚87762120014848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚89855643663708 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4476305957360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 % _timestamp‚1.7858124129587867e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53852853 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚87555016981504 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14602708 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚87555034020864 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚85396180721152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33890522 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53873046 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚87779327131648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4477183430694 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33890522 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14605533 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚92388914160640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4488169221632 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4609418875392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87753896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34690567 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4609562125312 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87763568 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚87779352035328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚89873364151846 % _timestamp‚1.7858124133091266e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53873046 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4488169221632 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34690313 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 143249920 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4488081784832 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53863376 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚85396163681792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4477097627458 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87753896 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33890520 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20085032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14605281 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚92388746007040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚87572183744512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4488081784832 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚92060265529344 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53863376 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚89873261309250 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚5099 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20085034 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251’mvŸÍ&û ÇM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚85396180721152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33890522 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53873046 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚87779327131648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4477183430694 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33890522 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14605533 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚92388914160640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4488169221632 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4609418875392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87753896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34690567 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4609562125312 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87763568 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚87779352035328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚89873364151846 % _timestamp‚1.7858124133091266e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53873046 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4488169221632 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34690313 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 143249920 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4488081784832 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53863376 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚85396163681792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4477097627458 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87753896 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33890520 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20085032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14605281 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚92388746007040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚87572183744512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4488081784832 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚92060265529344 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 168153600 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53863376 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚89873261309250 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚5099 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20085034 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87763568 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚87572200783872 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320–Œ¦2ü - _wandbruntime‚2158  _runtime‚2158ÙR:0ý ü 0loss/FBt1/assignment_loss‚0.5067765479060736 .loss/FBt1/detection_loss‚1.779744646765969 epoch‚0 % _timestamp‚1.7858124133103273e+09 _step‚407 0loss/FRt1/assignment_loss‚3.2328842878341675 0loss/FRt2/assignment_loss‚3.6499024188681823 /loss/FRt2/detection_loss‚2.7891243937080854 trainer/global_step‚5099 -loss/FRt1/detection_loss‚2.32433208078146 1loss/SRqqt1/detection_loss‚0.9039779771391865 /loss/FBt2/detection_loss‚1.4270081398023682 %loss/total_loss‚21.97342394470014 1loss/SRqqt1/assignment_loss‚1.474912554758494 1loss/FBt2/assignment_loss‚0.11870534070805405 1loss/SRqqt2/detection_loss‚2.0392317355920873 _runtime‚2158.500081257 2loss/SRqqt2/assignment_loss‚1.7268238208360143§¦Ä=Yþ  -loss/FRt1/detection_loss‚2.32433208078146 1loss/SRqqt1/detection_loss‚0.9039779771391865 /loss/FBt2/detection_loss‚1.4270081398023682 %loss/total_loss‚21.97342394470014 1loss/SRqqt1/assignment_loss‚1.474912554758494 1loss/FBt2/assignment_loss‚0.11870534070805405 1loss/SRqqt2/detection_loss‚2.0392317355920873 _runtime‚2158.500081257 2loss/SRqqt2/assignment_loss‚1.7268238208360143 0loss/FBt1/assignment_loss‚0.5067765479060736 .loss/FBt1/detection_loss‚1.779744646765969 epoch‚0 % _timestamp‚1.7858124133103273e+09 _step‚407 0loss/F“ÇŽ‘®Rt1/assignment_loss‚3.2328842878341675 0loss/FRt2/assignment_loss‚3.6499024188681823 /loss/FRt2/detection_loss‚2.7891243937080854 trainer/global_step‚5099—wóvpœÿ ‚(:‘ ¾«ÅÓ»„Ÿƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚101.626' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚33.87533333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0™¨©M`€‚(:Õ  ¾«ÅÓàäɃ) proc.memory.availableMB‚ 38512.7421875 disk.drbd2041.out‚0 disk.sda.out‚72.96875 disk.rbd0.out‚0 disk.drbd2111.out‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚2.46875# proc.memory.rssMB‚ 11253.8828125 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.rbd1.out‚ 8.7216796875 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.drbd1550.out‚0 disk.drbd1828.out‚0! disk.dm-0.out‚613.37939453125 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd2118.in‚0 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.sda1.out‚0$ disk.nvme0n1.out‚613.37939453125$ memory_percent‚41.23421907424927* proc.memory.percent‚17.172062397003174 network.sent‚ 4.730712e+06 disk.drbd1356.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚2.46875 disk.sda1.in‚0 disk.sda2.out‚72.96875 cpu‚13.809832810923872 disk.drbd2041.in‚0 disk.sda.in‚ 0.68359375 disk.drbd2078.in‚0 disk.drbd2078.out‚0 disk.drbd1552.out‚0 disk.rbd1.in‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd2118.out‚0 network.recv‚345964 disk.drbd1931.in‚0 proc.cpu.threads‚29 disk.drbd1285.in‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd1226.in‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1828.in‚0) disk./.usagePercent‚32.79180153600515 disk.drbd1584.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0# disk./.usageGB‚586.350715637207÷æcׂ(:’ Í«ÅÓŠ¿‹ƒ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚115.366' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.455333333333336 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¶šªí\‚‚(:Ñ  Í«ÅÓý•Óƒ network.sent‚ 4.775661e+06 disk.sda.in‚ 0.68359375 disk.drbd1965.out‚0 disk.drbd2111.in‚0 disk.drbd1289.out‚0 disk.rbd1.in‚0 disk.drbd1817.in‚0 disk.drbd1673.out‚0 cpu‚13.808872365382573 disk.drbd2078.in‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0 disk.rbd1.out‚ 8.7216796875) disk./.usagePercent‚32.79180153600515 disk.sda.out‚73.625 disk.drbd1550.in‚0 disk.drbd2078.out‚0 disk.drbd1356.in‚0 disk.drbd1552.out‚0 disk.drbd1931.in‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.sda1.in‚0$ disk.nvme0n1.out‚613.56689453125 disk.sda1.out‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.drbd2118.in‚0* proc.memory.percent‚17.172062397003174 disk.drbd1584.out‚0 disk.drbd1285.out‚0 disk.dm-0.in‚2.46875 disk.drbd2118.out‚0 disk.drbd1673.in‚0 disk.sda2.out‚73.625 disk.drbd1584.in‚0 disk.sda2.in‚ 0.68359375) proc.memory.availableMB‚ 38512.4453125# disk./.usageGB‚586.350715637207 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.rbd0.out‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0# proc.memory.rssMB‚ 11253.8828125 disk.drbd1356.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd2041.in‚0 disk.drbd1550.out‚0$ memory_percent‚41.23467206954956 proc.cpu.threads‚29 network.recv‚348382 disk.drbd2111.out‚0 disk.drbd1552.in‚0! disk.dm-0.out‚613.56689453125 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1817.out‚0 disk.drbd1938.out‚0ÓN‘E2ƒ- _wandbruntime‚2179  _runtime‚2179?>ۙƄÀ _runtime‚ 2179.16239111 _step‚408 & lr-AdamW/pg1‚0.0001327009882128411 & lr-AdamW/pg2‚0.0001327009882128411 trainer/global_step‚5149 % _timestamp‚1.7858124339724727e+09“˻˅Š& lr-AdamW/pg1‚0.0001327009882128411 & lr-AdamW/pg2‚0.0001327009882128411 trainer/global_step‚5149 % _timestamp‚1.7858124339724727e+09 _runtime‚ 2179.16239111 _step‚408˜O×:š2†- _wandbruntime‚2179  _runtime‚2179PKÌá¾'‡¸O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚92944657901568 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54390511 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚90736643585700 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚93276343410688 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚88596882 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚88413355131904 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35023524 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54380855 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35023277 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚86216432125472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34216027 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4531198300672 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4520194420868 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34216027 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚88622500758528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚88413372171264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14745577 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34216029 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4531198300672 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚88622475854848 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4653842652160 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚2179.166616096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚92944553432576 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚92944657901568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54390511 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20277945 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚92944553432576 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34216029 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚86216449164832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚88413372171264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124ˆƒcSÃ'ˆ½O QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚86216432125472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34216027 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4531198300672 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4520194420868 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34216027 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚88622500758528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚88413372171264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14745577 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34216029 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4531198300672 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚88622475854848 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4653842652160 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚2179.166616096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚92944553432576 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚92944657901568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54390511 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20277945 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚92944553432576 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34216029 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚86216449164832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚88413372171264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚92944657901568 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54390511 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚90736643585700 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚93276343410688 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚88596882 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚88413355131904 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35023524 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54380855 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35023277™ª{·!2‰- _wandbruntime‚2179  _runtime‚2179Ší^ZŠÁM [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93294453535232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 _step‚410 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚88430521894912 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88623720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 % _timestamp‚1.7858124343318498e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88614048 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88623720 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34222672 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 2179.52216816 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚92962599815168 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚92962704291328 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93294302158848 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4532165357056 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚236 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviŸÈüÃm ceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚86233189337152 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54401048 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚88430521894912 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚88430538934272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚92962704291328 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4532077920256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20281882 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34222670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4532165357056 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35030048èuÌ&‹ÆM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35030286 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54391378 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚238 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4521071894202 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 trainer/global_step‚5149 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚90754261231354 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚88430521894912 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88623720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 % _timestamp‚1.7858124343318498e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88614048 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88623720 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34222672 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 2179.52216816 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚92962599815168 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚92962704291328 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93294302158848 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4532165357056 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚236 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚86233189337152 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54401048 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚88430521894912 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚88430538934272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚92962704291328 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4532077920256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20281882 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34222670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4532165357056 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35030048 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93294453535232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 _step‚410 ODeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93846887718400 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14889473 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚87053457780832 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚93846992187392 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚87053440741472 % _timestamp‚1.7858124547254539e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚89271693282304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89466692 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54918513 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35364527 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35364268 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20474795 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚94181947650048 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚89271710321664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34548179 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4575194436096 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚93846992187392 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚94181781586432 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚93846887718400 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚91617540665208 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚89271693282304 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4698949891584 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4563997081252 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14889730 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4564082884376 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34548177 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚91617437822724 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54908857 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4699091051520 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2199.916354815 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚89482856598528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20474797 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34548179 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚89271710321664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658wp…–½O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54918513 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93846887718400 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14889473 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚87053457780832 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚93846992187392 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚87053440741472 % _timestamp‚1.7858124547254539e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚89271693282304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89466692 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54918513 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35364527 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35364268 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20474795 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚94181947650048 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚89271710321664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34548179 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4575194436096 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚93846992187392 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚94181781586432 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚93846887718400 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚91617540665208 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚89271693282304 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4698949891584 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4563997081252 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14889730 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4564082884376 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34548177 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚91617437822724 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54908857 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4699091051520 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runÜ?s…>time‚2199.916354815 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚89482856598528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20474797 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34548179 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚89271710321664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658÷r%2—- _wandbruntime‚2200  _runtime‚2200µ:y†È&˜ÂM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89483872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4576161492480 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚89500063715328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35371132 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚94199893045248 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚91635055468266 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34554822 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4564874554474 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4699854233600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89483872 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚93865038577152 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34554822 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54929050 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35371387 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89474200 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4576074055680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4564960357710 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34554820 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚89474200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54919380 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚89288860045312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87070180913792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 _runtime‚2200.261049116 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚91635158310862 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚87070197953152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚93864934100992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20478734 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93865038577152 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54919380 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20478732 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚89288877084672 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚94200048615936 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 % _timestamp‚1.7858124550707445e+09 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚89288877084672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4576074055680P÷_èÍ&™ÇM C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89474200 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4576074055680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4564960357710 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34554820 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚89474200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54919380 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚89288860045312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87070180913792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 _runtime‚2200.261049116 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚91635158310862 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚87070197953152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚93864934100992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20478734 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93865038577152 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54919380 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20478732 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚89288877084672 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚94200048615936 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 % _timestamp‚1.7858124550707445e+09 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚89288877084672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4576074055680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89483872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4576161492480 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚89500063715328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35371132 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚94199893045248 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚91635055468266 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34554822 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4564874554474 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4699854233600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89483872 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚93865038577152 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34554822 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54929050 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35371387 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640žu†io2š- _wandbruntime‚2200  _runtime‚2200n‹›‚ 1loss/SRqqt2/detection_loss‚2.0317426978007838 _step‚415 0loss/FRt2/assignment_loss‚3.3525173316399255 2loss/SRqqt1/assignment_loss‚1.4278128163344195 2loss/SRqqt2/assignment_loss‚1.2868729177656968 .loss/FBt2/detection_loss‚6.051687005015108 &loss/total_loss‚25.870311486008877 0loss/FRt1/assignment_loss‚3.7830193200532127 2loss/FBt2/assignment_loss‚0.036752764153156176 /loss/FRt1/detection_loss‚2.5088692792855642 /loss/FBt1/detection_loss‚1.8717083268695407 % _timestamp‚1.7858124550722837e+09 1loss/FBt1/assignment_loss‚0.20789946807619547 /loss/FRt2/detection_loss‚2.4607049270222583 1loss/SRqqt1/detection_loss‚0.8507246319930177 epoch‚0 trainer/global_step‚5199 _runtime‚2200.262077751È 7Å œ‡ 0loss/FRt1/assignment_loss‚3.7830193200532127 2loss/FBt2/assignment_loss‚0.036752764153156176 /loss/FRt1/detection_loss‚2.5088692792855642 /loss/FBt1/detection_loss‚1.8717083268695407 % _timestamp‚1.7858124550722837e+09 1loss/FBt1/assignment_loss‚0.20789946807619547 /loss/FRt2/detection_loss‚2.4607049270222583 1loss/SRqqt1/detection_loss‚0.8507246319930177 epoch‚0 trainer/global_step‚5199 _runtime‚2200.262077751 1loss/SRqqt2/detection_loss‚2.0317426978007838 _step‚415 0loss/FRt2/assignment_loss‚3.3525173316399255 2loss/SRqqt1/assignment_loss‚1.4278128163344195 2loss/SRqqt2/assignment_loss‚1.2868729177656968 .loss/FBt2/detection_loss‚6.051687005015108 &loss/total_loss‚25.870311486008877ŸŠ² ’œ‚(:‘ ë«ÅÓ…²æ‚ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚111.403' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.13433333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0á?hž‚(:Ý  ë«ÅÓñ̪ƒ$ disk./.usageGB‚586.3508338928223 disk.rbd1.in‚0 disk.drbd1828.in‚0 disk.drbd1817.out‚0 disk.sda.out‚ 74.43359375 disk.drbd1550.out‚0 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.drbd1938.out‚0% memory_percent‚41.234761476516724* disk./.usagePercent‚32.791808149478214 disk.sda.in‚ 0.68359375 disk.dm-0.out‚614.0322265625 cpu‚13.806719823442133 disk.sda2.in‚ 0.68359375* proc.memory.availableMB‚38512.38671875 disk.drbd1552.in‚0 disk.drbd2118.out‚0* proc.memory.percent‚17.172062397003174 proc.cpu.threads‚29# disk.nvme0n1.out‚614.0322265625 disk.drbd1584.in‚0 disk.drbd1285.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1931.out‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1552.out‚0 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.drbd2111.in‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd1965.out‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.sda2.out‚ 74.43359375 disk.rbd0.in‚0 disk.sda1.in‚0 network.recv‚351876 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.dm-0.in‚2.46875 disk.drbd1938.in‚0# proc.memory.rssMB‚ 11253.8828125 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.rbd1.out‚ 8.9716796875 disk.rbd0.out‚0 network.sent‚ 4.823635e+06 disk.nvme0n1.in‚2.46875 disk.drbd2078.out‚0 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0'b‘,‘Ÿ‚(:† ú«ÅÓÊž«ƒ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚109.269' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚36.423 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0èèÿf] ‚(:Ò  ú«ÅÓðõ׃ disk.drbd1931.out‚0 disk.sda2.out‚74.8125 network.sent‚ 4.868847e+06 disk.drbd2111.in‚0 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.rbd1.out‚ 9.0341796875 disk.drbd2041.in‚0 disk.sda.out‚74.8125 disk.drbd1550.in‚0 disk.dm-0.in‚2.46875 disk.dm-0.out‚ 614.794921875 disk.drbd1289.out‚0 disk.rbd0.out‚0* disk./.usagePercent‚32.791808149478214 disk.drbd1510.out‚0 disk.drbd1550.out‚0 network.recv‚354328 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.drbd2078.out‚0 disk.sda1.out‚0 disk.drbd1584.in‚0 proc.cpu.threads‚29 disk.sda2.in‚ 0.68359375 disk.drbd1931.in‚0$ memory_percent‚41.23451113700867 disk.drbd1828.in‚0 disk.drbd2118.in‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd1226.out‚0$ disk./.usageGB‚586.3508338928223 disk.drbd1817.out‚0 disk.drbd1673.in‚0" disk.nvme0n1.out‚ 614.794921875* proc.memory.percent‚17.172324657440186 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd1965.in‚0# proc.memory.rssMB‚ 11254.0546875 disk.drbd2111.out‚0 disk.drbd1356.in‚0* proc.memory.availableMB‚38512.55078125 disk.drbd2078.in‚0 cpu‚13.805521432304456 disk.drbd2041.out‚0 disk.sda.in‚ 0.68359375 disk.rbd1.in‚0 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.drbd1289.in‚0 disk.drbd1552.in‚0 disk.nvme0n1.in‚2.46875&T 2¡- _wandbruntime‚2220  _runtime‚2220#ºâ—ɢà ' lr-AdamW/pg2‚0.00013527820686137172 trainer/global_step‚5249 % _timestamp‚1.7858124756099594e+09 _runtime‚2220.799914832 _step‚416 ' lr-AdamW/pg1‚0.00013527820686137172)0¹;Î£È % _timestamp‚1.7858124756099594e+09 _runtime‚2220.799914832 _step‚416 ' lr-AdamW/pg1‚0.00013527820686137172 ' lr-AdamW/pg2‚0.00013527820686137172 trainer/global_step‚5249 ºÿÿ2¤- _wandbruntime‚2220  _runtime‚2220ö­íW ¥¶O ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚95087474634752 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚94749222004224 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4619190571520 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55436859 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚5249 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚90130048472064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34880327 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35705420 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚90343187534848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4744262196224 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55446515 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 $ _timestamp‚1.785812475610931e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚90130031432704 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20671647 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4607971347884 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55446515 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚90317186 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35705160 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4744135716352 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚87890449357472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55436859 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20671645 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15033773 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚90130031432704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚94749326473216 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚90343212438528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚94749326473216 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚95087323251200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4607885544760 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚90130048472064 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15033515¨òÁ'¦»O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55446515 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚90317186 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35705160 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4744135716352 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚87890449357472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55436859 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20671645 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15033773 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚90130031432704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚94749326473216 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚90343212438528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚94749326473216 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚95087323251200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4607885544760 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚90130048472064 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15033515 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚95087474634752 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚94749222004224 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4619190571520 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55436859 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚5249 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚90130048472064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34880327 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35705420 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚90343187534848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4744262196224 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55446515 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 $ _timestamp‚1.785812475610931e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚90130031432704 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20671647 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4607971347884 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0¡x§2§- _wandbruntime‚2221  _runtime‚2221{hXÁÈ&¨ÂM REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20675582 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55457052 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚90360419555328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4620070191104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90344024 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚90147198195712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚90147215235072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚90147198195712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34886972 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4745033930752 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90334352 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55447382 trainer/global_step‚5249 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚94767372862976 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15036302 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4608763017982 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚92516055390370 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚95105590444544 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35712131 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚94767372862976 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚94767268386816 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35711884 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90344024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4745170889216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87907189529792 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90334352 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55447382 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚90147215235072 _step‚418 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7858124759599833e+09 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4620157627904 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55457052 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34886972 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20675584 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚90360394651648 _runtime‚2221.150255601 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34886970 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4608848821218©^ó ½©ÇM O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4620157627904 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55457052 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34886972 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20675584 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚90360394651648 _runtime‚2221.150255601 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34886970 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4608848821218 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20675582 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55457052 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚90360419555328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4620070191104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90344024 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚90147198195712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚90147215235072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚90147198195712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34886972 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4745033930752 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90334352 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55447382 trainer/global_step‚5249 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚94767372862976 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15036302 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4608763017982 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚92516055390370 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚95105590444544 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35712131 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚94767372862976 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚94767268386816 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35711884 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90344024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4745170889216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87907189529792 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90334352 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55447382 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚90147215235072 _step‚418 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7858124759599833e+09¢œŒJÙ2ª- _wandbruntime‚2221  _runtime‚2221Ä9Ö«ü .loss/FBt2/detection_loss‚1.781917716176927 % _timestamp‚1.7858124759611871e+09 _step‚419 /loss/FRt1/assignment_loss‚3.728180130807365 0loss/FBt1/assignment_loss‚0.2707018387154676 1loss/SRqqt1/assignment_loss‚1.317411404046597 1loss/FBt2/assignment_loss‚0.04054559205329156 /loss/FRt2/detection_loss‚2.5078559833011407 1loss/SRqqt1/detection_loss‚0.9459641515718293 epoch‚0 0loss/FRt2/assignment_loss‚3.4979484793783606 .loss/FRt1/detection_loss‚2.413691107092834 /loss/FBt1/detection_loss‚1.7777503132844752 %loss/total_loss‚21.70446376095601 trainer/global_step‚5249 _runtime‚2221.150970799 2loss/SRqqt2/assignment_loss‚1.4548313974613143 1loss/SRqqt2/detection_loss‚1.9676656470664087ªþŸ¬ 0loss/FBt1/assignment_loss‚0.2707018387154676 1loss/SRqqt1/assignment_loss‚1.317411404046597 1loss/FBt2/assignment_loss‚0.04054559205329156 /loss/FRt2/detection_loss‚2.5078559833011407 1loss/SRqqt1/detection_loss‚0.9459641515718293 epoch‚0 0loss/FRt2/assignment_loss‚3.4979484793783606 .loss/FRt1/detection_loss‚2.413691107092834 /loss/FBt1/detection_loss‚1.7777503132844752 %loss/total_loss‚21.70446376095601 trainer/global_step‚5249 _runtime‚2221.150970799 2loss/SRqqt2/assignment_loss‚1.4548313974613143 1loss/SRqqt2/detection_loss‚1.9676656470664087 .loss/FBt2/detection_loss‚1.781917716176927 % _timestamp‚1.7858124759611871e+09 _step‚419 /loss/FRt1/assignment_loss‚3.728180130807365£à ™d­‚(:Ù  ‰¬ÅÓŽÛÓƒ disk.drbd1938.out‚0 disk.rbd1.out‚ 9.0341796875* proc.memory.availableMB‚38512.48828125 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd1356.out‚0 proc.cpu.threads‚29 disk.dm-0.in‚2.46875 disk.sda1.in‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.rbd0.out‚0* proc.memory.percent‚17.172324657440186 disk.drbd1285.out‚0 disk.drbd1828.out‚0 disk.nvme0n1.in‚2.46875 disk.sda.in‚ 0.68359375 disk.drbd1285.in‚0 disk.sda2.in‚ 0.68359375$ disk./.usageGB‚586.3508338928223 disk.drbd1289.in‚0 disk.sda2.out‚ 75.29296875 disk.drbd1584.in‚0 network.recv‚355365 disk.drbd1828.in‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0* disk./.usagePercent‚32.791808149478214 disk.drbd1510.in‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.drbd1550.out‚0 disk.dm-0.out‚ 614.978515625 disk.drbd2118.in‚0 disk.drbd1356.in‚0 disk.rbd0.in‚0$ memory_percent‚41.23460650444031 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0# proc.memory.rssMB‚ 11254.0546875 disk.drbd2111.in‚0 disk.drbd2078.out‚0 cpu‚13.804778724620405 network.sent‚ 4.87186e+06" disk.nvme0n1.out‚ 614.978515625 disk.sda1.out‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd2111.out‚0 disk.drbd2041.out‚0 disk.sda.out‚ 75.29296875 disk.drbd1965.in‚0FJª]œ®‚(:‘ ‰¬ÅÓ¤çу gpu.0.gpu‚43.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚116.933' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.97766666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0@·†¦2¯- _wandbruntime‚2241  _runtime‚2241•ïþàÅ°¿ % lr-AdamW/pg2‚0.000136566816185637 trainer/global_step‚5299 % _timestamp‚1.7858124965287502e+09 _runtime‚2241.718697896 _step‚420 % lr-AdamW/pg1‚0.000136566816185637D¦Ê±Ä _runtime‚2241.718697896 _step‚420 % lr-AdamW/pg1‚0.000136566816185637 % lr-AdamW/pg2‚0.000136566816185637 trainer/global_step‚5299 % _timestamp‚1.7858124965287502e+09¤¿Ÿu@2²- _wandbruntime‚2241  _runtime‚2241YàD=¾'³¸O `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚91203543374848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4789476004352 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚95651556290048 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15176177 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4789353718784 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4663186706944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20868497 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚95651556290048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚88727475012832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36044672 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚421 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚231 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚91203568278528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35212479 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4663274136576 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚95651660759040 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36044903 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚95992897093632 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55974517 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4663274136576 _runtime‚2241.723645308 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35212479 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚229 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚5299 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚95651660759040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15176406 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚90988369583104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚93379334824224 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4651859811392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚90988386622464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚90988369583104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55974517 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55964861 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20868495 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚95993044282880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858124965296643e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816ß>LÃ'´½O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4663186706944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20868497 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚95651556290048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚88727475012832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36044672 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚421 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚231 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚91203568278528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35212479 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4663274136576 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚95651660759040 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36044903 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚95992897093632 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55974517 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4663274136576 _runtime‚2241.723645308 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35212479 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚229 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚5299 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚95651660759040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15176406 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚90988369583104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚93379334824224 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4651859811392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚90988386622464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚90988369583104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55974517 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55964861 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20868495 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚95993044282880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858124965296643e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚91203543374848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4789476004352 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚95651556290048 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15176177 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4789353718784 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0¥4B¿l2µ- _wandbruntime‚2242  _runtime‚2242i,•â­ ¶ÂM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35219120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36051410 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚93396849627282 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91204176 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35219122 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚96011165581312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91204176 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4664153763328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15179228 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4664153763328 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚96011007913472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚93396952469878 H@DeviceStatsÔ@™±Monitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858124968770342e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚91005553385472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35219120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20872434 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚95669602672640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4652651481490 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4664066326528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91194504 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20872432 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚88744215185152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚91005536346112 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15178978 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚5299 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚91005536346112 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4664066326528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚88744198145792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚91220750491648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4790257421824 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚2242.067332302 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55975384 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚422 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚91005553385472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4790390185984 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91194504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚95669707148800zFâ7Í&·ÇM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚91005553385472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35219120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20872434 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚95669602672640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4652651481490 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4664066326528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91194504 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20872432 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚88744215185152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚91005536346112 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15178978 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚5299 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚91005536346112 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4664066326528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚88744198145792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚91220750491648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4790257421824 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚2242.067332302 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55975384 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚422 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚91005553385472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4790390185984 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91194504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚95669707148800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35219120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36051410 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚93396849627282 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91204176 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35219122 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚96011165581312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91204176 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4664153763328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15179228 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4664153763328 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚96011007913472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚93396952469878 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858124968770342e+09¦¶¶ó2¸- _wandbruntime‚2242  _runtime‚2242Èÿóÿ¹ü 2loss/SRqqt2/assignment_loss‚1.5527107474895625 1loss/FBt1/assignment_loss‚0.17964604692533612 1loss/FBt2/assignment_loss‚0.11188638838910213 .loss/FBt2/detection_loss‚4.950320653872745 trainer/global_step‚5299 0loss/FRt1/assignment_loss‚3.8192087485240056 1loss/SRqqt1/detection_loss‚1.0343568886690322 /loss/FBt1/detection_loss‚2.4206560254096985 _step‚423 2loss/SRqqt1/assignment_loss‚1.4895426284281477 .loss/FRt1/detection_loss‚2.612399451319988 .loss/FRt2/detection_loss‚2.515400282189816 1loss/SRqqt2/detection_loss‚2.1135628471007712 epoch‚0 _runtime‚2242.068028777 /loss/FRt2/assignment_loss‚3.564361104482337 %loss/total_loss‚26.36405181280054 $ _timestamp‚1.785812496878256e+09ñ0Pº 2loss/SRqqt2/assignment_loss‚1.5527107474895625 1loss/FBt1/assignment_loss‚0.17964604692533612 1loss/FBt2/assignment_loss‚0.11188638838910213 .loss/FBt2/detection_loss‚4.950320653872745 trainer/global_step‚5299 0loss/FRt1/assignment_loss‚3.8192087485240056 1loss/SRqqt1/detection_loss‚1.0343568886690322 /loss/FBt1/detection_loss‚2.4206560254096985 _step‚423 2loss/SRqqt1/assignment_loss‚1.4895426284281477 .loss/FRt1/detection_loss‚2.612399451319988 .loss/FRt2/detection_loss‚2.515400282189816 1loss/SRqqt2/detection_loss‚2.1135628471007712 epoch‚0 _runtime‚2242.068028777 /loss/FRt2/assignment_loss‚3.564361104482337 %loss/total_loss‚26.36405181280054 $ _timestamp‚1.785812496878256e+09§­kf»‚(:Û  ˜¬ÅÓìÀ¬ƒ network.sent‚ 4.916916e+06$ disk./.usageGB‚586.3508529663086 disk.nvme0n1.in‚2.46875$ disk.nvme0n1.out‚615.24365234375 disk.drbd2111.out‚0 disk.drbd1584.in‚0 disk.rbd1.out‚ 9.06640625 network.recv‚357945 disk.drbd1938.in‚0 disk.rbd0.in‚0 disk.drbd2118.out‚0% memory_percent‚41.235142946243286* proc.memory.availableMB‚38512.13671875 disk.drbd1584.out‚0 disk.drbd1285.in‚0 disk.dm-0.in‚2.46875 proc.cpu.threads‚29 disk.sda2.in‚ 0.68359375 disk.drbd2041.out‚0 disk.drbd1828.out‚0 disk.drbd2041.in‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd1289.out‚0 disk.sda.out‚ 75.7890625 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1828.in‚0 disk.drbd1817.in‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0* disk./.usagePercent‚32.791809216167415 disk.drbd1673.in‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1552.out‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0! disk.dm-0.out‚615.24365234375 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.drbd1965.in‚0 disk.drbd2111.in‚0 disk.sda1.in‚0 disk.sda2.out‚ 75.7890625# proc.memory.rssMB‚ 11254.0546875 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.drbd1552.in‚0 disk.rbd1.in‚0 disk.sda.in‚ 0.68359375 disk.drbd2078.in‚0 disk.sda1.out‚0 disk.drbd1938.out‚0 disk.drbd2078.out‚0 cpu‚13.803832836230457* proc.memory.percent‚17.172324657440186 Mœ¼‚(:‘ ˜¬ÅÓ®çêƒ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚111.172' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.05733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÕLª2½- _wandbruntime‚2262  _runtime‚2262i¹[ǾÁ & lr-AdamW/pg2‚0.0001378554255099023 trainer/global_step‚5349 % _timestamp‚1.7858125175485787e+09 _runtime‚2262.738523601 _step‚424 & lr-AdamW/pg1‚0.0001378554255099023o¶‰Ì¿Æ & lr-AdamW/pg1‚0.0001378554255099023 & lr-AdamW/pg2‚0.0001378554255099023 trainer/global_step‚5349 % _timestamp‚1.7858125175485787e+09 _runtime‚2262.738523601 _step‚424¨¦¡;2À- _wandbruntime‚2262  _runtime‚2262ÔWÕ¾'Á¸O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56502519 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36385033 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _step‚425 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35544627 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35544629 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35544627 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚270 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚96553890575872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15319686 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚94260231903732 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4707270272000 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4834527831040 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚96553890575872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56492863 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚96553995044864 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚89564466589472 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4695662471776 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36384761 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21065347 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4695748274900 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35544629 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4834664796672 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚91846724772864 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56492863 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚94260129061248 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚5349 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚91846707733504 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21065345 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4707270272000 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858125175494819e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4707182842368ãd¶á˽O \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚91846724772864 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56492863 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚94260129061248 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚5349 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚91846707733504 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21065345 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_tra» 0ø!in_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4707270272000 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858125175494819e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4707182842368 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56502519 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36385033 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _step‚425 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35544627 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35544629 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35544627 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚270 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚96553890575872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15319686 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚94260231903732 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4707270272000 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4834527831040 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚96553890575872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56492863 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚96553995044864 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚89564466589472 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4695662471776 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36384761 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21065347 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4695748274900 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35544629 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4834664796672©ÝÂ@2Ã- _wandbruntime‚2263  _runtime‚2263À:sÈ&ÄÂM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21069282 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4835568852992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21069284 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92054656 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚91863874496512 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92064328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚94277746706790 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4708062461952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚91863891535872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚91863874496512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15322318 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4708149898752 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚92081131235328 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚96571936958464 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚92054656 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚96916536129024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92064328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4696539944998 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92081106331648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4696625748234 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚96572041434624 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚89581223801152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35551272 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚94277849549386 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15322571 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36391600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888„±å&Í&ÅÇM REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35551272 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36391855 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21069282 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4835568852992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21069284 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92054656 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚91863874496512 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92064328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚94277746706790 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4708062461952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚91863891535872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚91863874496512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15322318 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4708149898752 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚92081131235328 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚96571936958464 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚92054656 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚96916536129024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92064328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4696539944998 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92081106331648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4696625748234 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚96572041434624 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚89581223801152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35551272 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚94277849549386 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15322571 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36391600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FÚcÌ‚(:Ù  ¶¬ÅӾムd¥m¡HFisk.drbd1817.out‚0 disk.drbd2041.out‚0 disk.drbd1550.out‚0 disk.drbd1510.in‚0* disk./.usagePercent‚32.791808149478214 disk.drbd1938.in‚0 disk.rbd1.out‚ 9.30859375 disk.drbd2041.in‚0# proc.memory.rssMB‚ 11254.0546875 disk.drbd1817.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1965.out‚0* proc.memory.percent‚17.172324657440186 network.sent‚ 5.006863e+06 network.recv‚362779 disk.drbd1673.in‚0 disk.drbd1510.out‚0 disk.sda1.in‚0 disk.drbd1931.in‚0 disk.drbd1289.in‚0 cpu‚13.80154096022376 disk.drbd1226.out‚0 disk.drbd1584.out‚0 disk.drbd1552.in‚0 disk.drbd2078.out‚0 disk.drbd1931.out‚0 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.drbd1550.in‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.sda2.in‚ 0.68359375 disk.nvme0n1.in‚2.46875 disk.drbd1584.in‚0 disk.sda2.out‚ 76.4296875 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd2118.in‚0 disk.drbd1673.out‚0 disk.drbd1938.out‚0 disk.drbd1356.in‚0 disk.drbd1285.out‚0 disk.drbd1289.out‚0 proc.cpu.threads‚29* proc.memory.availableMB‚38512.91015625$ disk./.usageGB‚586.3508338928223 disk.sda.out‚ 76.4296875 disk.drbd1828.in‚0! disk.dm-0.out‚615.89501953125 disk.drbd1552.out‚0$ memory_percent‚41.23396277427673 disk.drbd1226.in‚0 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.drbd2111.out‚0$ disk.nvme0n1.out‚615.89501953125 disk.dm-0.in‚2.46875Ê[92Í- _wandbruntime‚2283  _runtime‚2283—äXkÉÎà ' lr-AdamW/pg1‚0.00013914403483416762 ' lr-AdamW/pg2‚0.00013914403483416762 trainer/global_step‚5399 % _timestamp‚1.7858125383424945e+09 _runtime‚2283.534618488 _step‚428I ÎÏÈ _runtime‚2283.534618488 _step‚428 ' lr-AdamW/pg1‚0.00013914403483416762 ' lr-AdamW/pg2‚0.00013914403483416762 trainer/global_step‚5399 % _timestamp‚1.7858125383424945e+09¬5W¨ß2Ð- _wandbruntime‚2283  _runtime‚2283 lÚ½'Ñ·O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚97456329330688 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35876777 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚92907300 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚90401475205472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35876779 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4751266407424 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4751178977792 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4879894175232 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚95141128983240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57030521 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚97804018555904 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚5399 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15463976 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚97456224861696 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4739636738408 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚97456224861696 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚232 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚97456329330688 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35876777 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚92924279958528çaO)Â'Ò¼O LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35876777 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚92924279958528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚97456329330688 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35876777 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚92907300 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚90401475205472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35876779 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4751266407424 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4751178977792 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4879894175232 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚95141128983240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57030521 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚97804018555904 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚5399 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15463976 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚97456224861696 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4739636738408 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚97456224861696 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚232 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚97456329330688 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0­A 1•2Ó- _wandbruntime‚2283  _runtime‚2283›¸ûáÇ&ÔÁM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4880813239296 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57041058 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚97822300314624 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚97474271244288 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4740428408506 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚92914808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚97474375720448 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21266134 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15466899 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚95158643786298 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4752146034176 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92924480 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92914808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35883420 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4752146034176 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35883422 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15466639 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4740514211742 _step‚430 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92941462171648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚95158746628894 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚5399 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 $ _timestamp‚1.785812538694549e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚92722212646912 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21266132 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚90418215377792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57031388 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36732771 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4752058597376 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36733033 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚2283.884884217 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92924480 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57041058 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36732771 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4752058597376 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36733033 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚2283.884884217 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92924480 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57041058 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚97474375720448 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21266134 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15466899 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚95158643786298 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4752146034176 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92924480 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92914808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35883420 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4752146034176 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35883422 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15466639 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4740514211742 _step‚430 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92941462171648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚95158746628894 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚5399 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 $ _timestamp‚1.785812538694549e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚92722212646912 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21266132 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚90418215377792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57031388 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37067396 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4924983068672 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21459047 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36208927 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57548867 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96022026062748 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _step‚433 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚93563384034304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37067151 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚98358663616512 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4795262542848 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 2304.59262112 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚98358559147520 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4795262542848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21459045 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚98358559147520 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4783439398792 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚93563384034304 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4783525201916 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚98709753735680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36208929 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚98358663616512 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15608349 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚91238500860832 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚5449 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57558523 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚93563401073664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36208929 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96021923220264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858125593994539e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93757794 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4795175113216 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚93784610894848 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚91238483821472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36208927 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4925117937152 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚93784635798528yèyÜ"à¼O D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96021923220264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 % _timestamp‚1.7858125593994539e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93757794 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4795175113216 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚93784610894848 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚91238483821472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36208927 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4925117937152 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚93784635798528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚98709593963520 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93767452 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37067396 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4924983068672 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21459047 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36208927 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57548867 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96022026062748 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _step‚433 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚93563384034304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37067151 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚98358663616512 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4795262542848 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 2304.59262112 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚98358559147520 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4795262542848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21459045 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚98358559147520 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4783439398792 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚93563384034304 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4783525201916 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚98709753735680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 G?DevicçZLÜÂeStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36208929 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚98358663616512 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15608349 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚91238500860832 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚5449 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57558523 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚93563401073664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36208929 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680±Y¤ ¦2á- _wandbruntime‚2304  _runtime‚2304ä¹a]Ç&âÁM ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚93580567836672 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚93801842915328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4926017866752 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36215572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57559390 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37073796 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4784316872014 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4796054732800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96039540865806 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4796142169600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚93580550797312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93784632 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚93580550797312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37074043 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21462984 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚434 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚98727860782080 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4925891394048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4796142169600 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚98376710006272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36215570 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15611059 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785812559751202e+09 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚5449 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚98376710006272 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93774960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36215570 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93784632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21462982 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57569060 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4796054732800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93774960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚91255241033152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4784402675250 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15610814 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚91255223993792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392¾CÌ&ãÆM [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚98727860782080 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4925891394048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4796142169600 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚98376710006272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36215570 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15611059 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785812559751202e+09 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚5449 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚98376710006272 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93774960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36215570 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93784632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21462982 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57569060 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4796054732800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93774960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚91255241033152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4784402675250 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15610814 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚91255223993792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚93580567836672 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚93801842915328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4926017866752 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36215572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57559390 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37073796 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4784316872014 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4796054732800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96039540865806 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4796142169600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚93580550797312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93784632 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚93580550797312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37074043 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21462984 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚434 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816²Å= y2ä- _wandbruntime‚2304  _runtime‚2304‹¾)ùåü 2loss/SRqqt1/assignment_loss‚1.2788193904223941 0loss/FBt1/assignment_loss‚0.2543559558689594 1loss/SRqqt1/detection_loss‚1.0648736878328267 trainer/global_step‚5449 $ _timestamp‚1.785812559752409e+09 _step‚435 0loss/FRt1/assignment_loss‚3.6229029046861747 2loss/SRqqt2/assignment_loss‚1.2867005929823905 -loss/FRt2/detection_loss‚2.31271780192852 /loss/FBt1/detection_loss‚1.0374600459097103 &loss/total_loss‚20.538376962541154 1loss/FBt2/assignment_loss‚0.25245858914359587 /loss/FRt1/detection_loss‚2.6163562057833922 /loss/FBt2/detection_loss‚1.2855215151518506 epoch‚0 /loss/FRt2/assignment_loss‚3.467377099990845 0loss/SRqqt2/detection_loss‚2.058833172840495 _runtime‚2304.942173774˜’+‹æ /loss/FRt1/detection_loss‚2.6163562057833922 /loss/FBt2/detection_loss‚1.2855215151518506 epoch‚0 /loss/FRt2/assignment_loss‚3.467377099990845 0loss/SRqqt2/detection_loss‚2.058833172840495 _runtime‚2304.942173774 2loss/SRqqt1/assignment_loss‚1.2788193904223941 0loss/FBt1/assignment_loss‚0.2543559558689594 1loss/SRqqt1/detection_loss‚1.0648736878328267 trainer/global_step‚5449 $ _timestamp‚1.785812559752409e+09 _step‚435 0loss/FRt1/assignment_loss‚3.6229029046861747 2loss/SRqqt2/assignment_loss‚1.2867005929823905 -loss/FRt2/detection_loss‚2.31271780192852 /loss/FBt1/detection_loss‚1.0374600459097103 &loss/total_loss‚20.538376962541154 1loss/FBt2/assignment_loss‚0.25245858914359587³ðäÏä›ç‚(: Ô¬ÅÓÚ«ªƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚129.53' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.17666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0pá[ùdè‚(:Ù  Ô¬ÅÓ ­¹ƒ disk.drbd1356.out‚0 disk.drbd1584.in‚0 disk.drbd1584.out‚0$ disk./.usageGB‚586.3527870178223 disk.drbd2111.out‚0 disk.sda2.out‚ 77.11328125 network.recv‚365946 disk.sda.in‚ 0.68359375 disk.rbd0.in‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0$ disk.nvme0n1.out‚616.45556640625 disk.sda.out‚ 77.11328125 disk.drbd1828.out‚0 cpu‚13.799678755978526 proc.cpu.threads‚29 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.drbd1285.in‚0! disk.dm-0.out‚616.45556640625 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd2041.in‚0 disk.drbd1552.out‚0$ memory_percent‚41.23373031616211# proc.memory.rssMB‚ 11254.0546875 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1938.out‚0 disk.sda1.in‚0 disk.drbd1931.in‚0 network.sent‚ 5.054833e+06 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.dm-0.in‚2.46875 disk.sda1.out‚0 disk.rbd1.out‚ 9.3408203125 disk.drbd1510.in‚0 disk.drbd1356.in‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0* proc.memory.percent‚17.172324657440186 disk.sda2.in‚ 0.68359375& proc.memory.availableMB‚ 38513.0625) disk./.usagePercent‚32.79191737845273 disk.drbd2078.in‚0 disk.drbd1817.in‚0¸’(‘é‚(:† ã¬ÅÓ¶ˆî‚ gpu.0.gpu‚37.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚125.376' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚41.792 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0VÎwhê‚(:Ý  ã¬ÅÓÖ¤´ƒ disk.drbd1356.in‚0 disk.drbd1289.in‚0# proc.memory.rssMB‚ 11254.0546875 network.sent‚ 5.099782e+06 disk.drbd1584.in‚0) disk./.usagePercent‚32.79191737845273 disk.drbd1285.out‚0 disk.drbd1356.out‚0 disk.drbd2078.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.drbd1938.out‚0$ disk.nvme0n1.out‚616.59228515625$ disk./.usageGB‚586.3527870178223 disk.drbd1289.out‚0 disk.drbd1938.in‚0* proc.memory.percent‚17.172324657440186 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1965.out‚0 disk.rbd0.out‚0 disk.drbd2111.out‚0 disk.drbd1828.in‚0 disk.sda2.in‚ 0.68359375 disk.rbd1.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd1550.in‚0 disk.drbd2111.in‚0 disk.drbd2118.in‚0 disk.rbd1.out‚ 9.4345703125$ memory_percent‚41.23449921607971 cpu‚13.798896107160182 network.recv‚368254 disk.drbd2041.in‚0 disk.sda1.out‚0 disk.sda.out‚ 77.41015625 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚2.46875 disk.drbd1510.in‚0 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd1285.in‚0 disk.drbd1931.in‚0! disk.dm-0.out‚616.59228515625 disk.sda2.out‚ 77.41015625 proc.cpu.threads‚29 disk.drbd1931.out‚0 disk.drbd2078.in‚0 disk.sda1.in‚0* proc.memory.availableMB‚38512.55859375 disk.drbd1817.in‚0 disk.drbd1965.in‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0ù2ë- _wandbruntime‚2325  _runtime‚2325ÛwÛ>Éìà ' lr-AdamW/pg2‚0.00014172125348269822 trainer/global_step‚5499 % _timestamp‚1.7858125801087062e+09 _runtime‚2325.300627701 _step‚436 ' lr-AdamW/pg1‚0.00014172125348269822·PÛáÎíÈ ' lr-AdamW/pg1‚0.00014172125348269822 ' lr-AdamW/pg2‚0.00014172125348269822 trainer/global_step‚5499 % _timestamp‚1.7858125801087062e+09 _runtime‚2325.300627701 _step‚436´“`¿Ù2î- _wandbruntime‚2325  _runtime‚23257f¶Ûï·O IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4839258678272 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚94627604 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21655897 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15750697 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚92075492437472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4827413665424 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96902923142256 _runtime‚ 2325.30072816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚99260893433344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36541079 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚99260893433344 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚94421739224064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4839171248640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37406348 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚94644966734848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚94421739224064 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚437 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚94644991638528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚99615169682432 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_bat&ù[Câch_start/segment.all.peak‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚94421722184704 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96902820299772 trainer/global_step‚5499 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚99615323163136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚94421722184704 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚99260997902336 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36541077 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58076869 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4970202947584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121ï8Â'ð¼O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21655895 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4827327862300 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37406594 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚99260997902336 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15750453 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94617946 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58086525 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58076869 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58086525 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36541079 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4970331524608 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚94421722184704 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96902820299772 trainer/global_step‚5499 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚99615323163136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚94421722184704 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚99260997902336 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36541077 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58076869 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4970202947584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4839258678272 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚94627604 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21655897 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15750697 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚92075492437472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4827413665424 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96902923142256 _runtime‚ 2325.30072816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚99260893433344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36541079 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚99260893433344 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚94421739224064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4839171248640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37406348 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚94644966734848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚94421739224064 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚437 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚94644991638528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚99615169682432 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚99633441020928 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚99279044292096 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21659834 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58087392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36547720 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚99633272867328 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96920437945314 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚94644784 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858125804514391e+09 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 143249920 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36547722 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36547722 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4828205335522 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4840138305024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚94438905987072 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94644784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚96920540787910 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21659832 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚2325.641701434 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚94635112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚92092232609792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4828291138758 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58097062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15753582 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4971099015680 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚94438888947712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15753333 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚5499 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚94635112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚99278939815936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37413165 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚94662198755328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4828205335522 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4840138305024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚94438905987072 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94644784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚96920540787910 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21659832 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚2325.641701434 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚94635112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚92092232609792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4828291138758 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58097062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15753582 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4971099015680 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚94438888947712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15753333 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚5499 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚94635112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚99278939815936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37413165 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚94662198755328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [HDeviceStatsMonitor.on_H[¬=Ž train_batch_end/inactive_split_bytes.all.allocated‚99633441020928 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚99279044292096 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21659834 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58087392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36547720 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚99633272867328 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96920437945314 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚94644784 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858125804514391e+09 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 143249920 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36547722 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36547722 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4883167384064 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4883254813696 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚100163227719168 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58614527 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21852747 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚100520708447232 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5015385872384 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58614527 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95487756 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4871216325808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚95280060335104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36873227 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4883254813696 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858126007885847e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚95280077374464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚97783820221764 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚100163332188160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4883167384064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚100520866122240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚100163332188160 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _runtime‚2345.981643556 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚95280060335104 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15893138 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚92912518092832 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36873227 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37745883 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15893396 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58604871 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚95505347478528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 trainer/global_step‚5549 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚92912501053472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚100163227719168 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58604871 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36873229 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37746143 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚100163227719168 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58604871 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36873229 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37746143 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4883167384064 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4883254813696 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚100163227719168 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58614527 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21852747 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚100520708447232 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5015385872384 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58614527 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95487756 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4871216325808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚95280060335104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36873227 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4883254813696 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858126007885847e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚95280077374464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚97783820221764 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚100163332188160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4883167384064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚100520866122240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚100163332188160 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 _runtime‚2345.981643556 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚95280060335104 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15893138 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚92912518092832¹Í-ŸX2ÿ- _wandbruntime‚2346  _runtime‚2346ö˜þÌ€ÇM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36879872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95495264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95504936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚95522554595328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4884134440448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚100181274101760 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95504936 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 $ _timestamp‚1.785812601128601e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58625064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118084096 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚95495264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚100538970856448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5016298177024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚97801437867418 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21856684 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚92929241225792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚95297244137472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4884047003648 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58625064 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚95297227098112 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 _step‚442 >6DeviceStatsMonitor.on_train_batch_end/num_dev‹1†¹¸ice_alloc‚251 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚97801335024822 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36879870 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 142987776 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4872093799030 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5016416261120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4884134440448 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚100181378577920 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚2346.318879531 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58615394 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15896196 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚95522529691648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4872179602266 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚95297244137472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚100181378577920 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37752880 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37752630 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚92929258265152 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36879870 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58615394 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4884047003648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21856682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121€rô™Ò&ÌM ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚92929258265152 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36879870 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58615394 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4884047003648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21856682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36879872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95495264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95504936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚95522554595328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4884134440448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚100181274101760 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95504936 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 $ _timestamp‚1.785812601128601e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58625064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118084096 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚95495264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚100538970856448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5016298177024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚97801437867418 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21856684 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚92929241225792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚95297244137472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4884047003648 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58625064 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚95297227098112 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 _step‚442 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚97801335024822 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36879870 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 142987776 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4872093799030 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5016416261120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4884134440448 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚100181378577920 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚2346.318879531 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58615394 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15896196 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚95522529691648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4872179602266 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚95297244137472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚100181378577920 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37752880 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37752630ºb,¿f2‚- _wandbruntime‚2346  _runtime‚2346í©O™ƒü /loss/FRt2/detection_loss‚2.5896187548230336 1loss/FBt1/assignment_loss‚0.38031760637467316 &loss/total_loss‚21.727341057572616 _runtime‚2346.319623263 /loss/FBt1/detection_loss‚1.5560566915406122 epoch‚0 1loss/SRqqt1/assignment_loss‚1.461768212720104 0loss/FBt2/assignment_loss‚0.1433305946466818 .loss/FRt1/detection_loss‚2.395087286284784 1loss/SRqqt1/detection_loss‚1.1823461312922123 % _timestamp‚1.7858126011298234e+09 trainer/global_step‚5549 _step‚443 1loss/SRqqt2/detection_loss‚1.9502684297530275 .loss/FBt2/detection_loss‚1.466009369597603 /loss/FRt1/assignment_loss‚3.802415621716802 /loss/FRt2/assignment_loss‚3.335311043553236 2loss/SRqqt2/assignment_loss‚1.4648113152698468©®É „ 1loss/SRqqt2/detection_loss‚1.9502684297530275 .loss/FBt2/detection_loss‚1.466009369597603 /loss/FRt1/assignment_loss‚3.802415621716802 /loss/FRt2/assignment_loss‚3.335311043553236 2loss/SRqqt2/assignment_loss‚1.4648113152698468 /loss/FRt2/detection_loss‚2.5896187548230336 1loss/FBt1/assignment_loss‚0.38031760637467316 &loss/total_loss‚21.727341057572616 _runtime‚2346.319623263 /loss/FBt1/detection_loss‚1.5560566915406122 epoch‚0 1loss/SRqqt1/assignment_loss‚1.461768212720104 0loss/FBt2/assignment_loss‚0.1433305946466818 .loss/FRt1/detection_loss‚2.395087286284784 1loss/SRqqt1/detection_loss‚1.1823461312922123 % _timestamp‚1.7858126011298234e+09 trainer/global_step‚5549 _step‚443»[Ìý…‚(:… ­ÅÓ¹¾ý‚ gpu.0.gpu‚22.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚97.881' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.627 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0+M:)g†‚(:Ü  ­ÅÓ¥üÁƒ disk.drbd1510.in‚0 disk.drbd2118.in‚0# proc.memory.rssMB‚ 11254.0546875 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.drbd1828.in‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd1550.in‚0 cpu‚13.797357689931149 disk.drbd1817.out‚0 disk.sda2.in‚ 0.68359375 disk.sda2.out‚ 78.04296875 disk.nvme0n1.in‚2.46875 disk.drbd1965.in‚0 disk.drbd1289.out‚0* proc.memory.percent‚17.172324657440186 disk.drbd2078.in‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0) proc.memory.availableMB‚ 38511.7421875 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.drbd1584.in‚0 network.sent‚ 5.147817e+06 disk.rbd1.out‚ 9.4970703125$ disk.nvme0n1.out‚617.20654296875 disk.drbd1584.out‚0 disk.drbd1550.out‚0 disk.drbd1828.out‚0! disk.dm-0.out‚617.20654296875 network.recv‚371812 disk.drbd1673.out‚0 disk.drbd1356.out‚0 disk.drbd1510.out‚0 disk.sda.in‚ 0.68359375 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd1226.in‚0 disk.dm-0.in‚2.46875 disk.sda.out‚ 78.04296875 disk.drbd1226.out‚0) disk./.usagePercent‚32.79191759179057 disk.sda1.in‚0 disk.drbd1673.in‚0 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 proc.cpu.threads‚29 disk.drbd1965.out‚0$ memory_percent‚41.23574495315552$ disk./.usageGB‚586.3527908325195 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0 disk.drbd1938.in‚0 disk.drbd1356.in‚0 disk.drbd2111.in‚0Öæo2‡- _wandbruntime‚2366  _runtime‚2366—Í{–Ɉà trainer/global_step‚5599 % _timestamp‚1.7858126216354158e+09 _runtime‚2366.825365415 _step‚444 ' lr-AdamW/pg1‚0.00014429847213122884 ' lr-AdamW/pg2‚0.00014429847213122884 ë€Î‰È trainer/global_step‚5599 % _timestamp‚1.7858126216354158e+09 _runtime‚2366.825365415 _step‚444 ' lr-AdamW/pg1‚0.00014429847213122884 ' lr-AdamW/pg2‚0.00014429847213122884¼ƒ"ª2Š- _wandbruntime‚2366  _runtime‚2366J¶­šÄ'‹¾O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚101426392893952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚93749509669472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59132873 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚93749526708832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚101065666473984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚96347908 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37205377 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37205377 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16037515 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101065562004992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5060558901248 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚242 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4915190592440 % _timestamp‚1.7858126216363494e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4927163519488 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚101065666473984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚101065562004992 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2366.829636913 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚96138415524864 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59132873 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4927250949120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22049597 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22049595 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚96138398485504 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38086870 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚96338250 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚96365678414848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37205377 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37205377 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16037515 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101065562004992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5060558901248 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚242 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4915190592440 % _timestamp‚1.7858126216363494e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4927163519488 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚101065666473984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚101065562004992 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2366.829636913 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚96138415524864 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59132873 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4927250949120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22049597 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22049595 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚96138398485504 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38086870 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚96338250 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚96365678414848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1½¨è 2- _wandbruntime‚2367  _runtime‚2367ý]xxÍ&ŽÇM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4928130575872 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4928130575872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚96382885531648 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4915982262538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96365088 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚96155565248512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96365088 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚98682334946926 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37212020 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4928043139072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4928043139072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37212020 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚93766249841792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59153066 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚101444506090496 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚101083608387584 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16040071 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4916068065774 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚93766266881152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚5599 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚ 2367.17994852 _step‚446 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38093603 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚96355416 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22053534 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96355416 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚96155582287872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59143396 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚96382910435328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5061595655168 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59153066 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38093857 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16040071 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4916068065774 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚93766266881152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚5599 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚ 2367.17994852 _step‚446 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38093603 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚96355416 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22053534 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96355416 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚96155582287872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59143396 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚96382910435328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5061595655168 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59153066 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38093857 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4915982262538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96365088 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚96155565248512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96365088 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚98682334946926 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37212020 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4928043139072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4928043139072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37212020 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚93766249841792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59153066 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚101444506090496 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936¾WÀSÿ2- _wandbruntime‚2367  _runtime‚2367Kב€ 0loss/FBt1/assignment_loss‚0.2085733131343659 /loss/FBt2/detection_loss‚2.0161439018350698 epoch‚0 trainer/global_step‚5599 _runtime‚2367.180839669 0loss/FRt2/assignment_loss‚3.6998974490672984 2loss/SRqqt1/assignment_loss‚1.3844163196004984 /loss/FRt1/detection_loss‚2.6832474598454104 1loss/SRqqt2/detection_loss‚2.1087626169834817 /loss/FBt1/detection_loss‚1.9640141800045967 &loss/total_loss‚22.642129015077533 0loss/FRt1/assignment_loss‚3.6748265897234282 % _timestamp‚1.7858126219908729e+09 _step‚447 2loss/SRqqt2/assignment_loss‚1.3770824790000915 1loss/FBt2/assignment_loss‚0.13843199173906787 .loss/FRt2/detection_loss‚2.505498284355123 1loss/SRqqt1/detection_loss‚0.8812344297891012Î8fÿ ’… 0loss/FBt1/assignment_loss‚0.2085733131343659 /loss/FBt2/detection_loss‚2.0161439018350698 epoch‚0 trainer/global_step‚5599 _runtime‚2367.180839669 0loss/FRt2/assignment_loss‚3.6998974490672984 2loss/SRqqt1/assignment_loss‚1.3844163196004984 /loss/FRt1/detection_loss‚2.6832474598454104 1loss/SRqqt2/detection_loss‚2.1087626169834817 /loss/FBt1/detection_loss‚1.9640141800045967 &loss/total_loss‚22.642129015077533 0loss/FRt1/assignment_loss‚3.6748265897234282 % _timestamp‚1.7858126219908729e+09 _step‚447 2loss/SRqqt2/assignment_loss‚1.3770824790000915 1loss/FBt2/assignment_loss‚0.13843199173906787 .loss/FRt2/detection_loss‚2.505498284355123 1loss/SRqqt1/detection_loss‚0.8812344297891012¿ßrœ“‚(:‘ ­ÅÓ’ …ƒ gpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚112.646' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.54866666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0aÅ·½`”‚(:Õ  ­ÅÓÎɃ disk.rbd0.out‚0 disk.drbd1584.in‚0 disk.drbd1289.in‚0 disk.drbd1552.in‚0 disk.sda2.in‚ 0.68359375$ disk./.usageGB‚586.3527946472168 disk.drbd1817.in‚0 disk.drbd1356.out‚0 disk.dm-0.out‚ 617.494140625 disk.drbd2041.in‚0 disk.sda.out‚ 78.3046875 disk.drbd1550.in‚0 cpu‚13.796705455125972 disk.drbd1965.in‚0 proc.cpu.threads‚29 disk.sda2.out‚ 78.3046875 disk.drbd1584.out‚0 disk.drbd2111.in‚0* proc.memory.percent‚17.172324657440186 disk.rbd1.in‚0$ memory_percent‚41.23486280441284) proc.memory.availableMB‚ 38512.3203125 network.recv‚374214) disk./.usagePercent‚32.79191780512841 disk.drbd1931.out‚0# proc.memory.rssMB‚ 11254.0546875 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.sda1.in‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd2118.in‚0 disk.drbd1356.in‚0 disk.sda.in‚ 0.68359375 network.sent‚ 5.192792e+06 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0 disk.drbd1673.out‚0" disk.nvme0n1.out‚ 617.494140625 disk.drbd2078.out‚0 disk.rbd1.out‚ 9.529296875 disk.drbd1965.out‚0 disk.drbd1285.out‚0 disk.drbd1552.out‚0 disk.drbd1931.in‚0 disk.drbd2078.in‚0 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd1938.out‚0 disk.drbd1550.out‚0 disk.drbd1828.in‚0 disk.dm-0.in‚2.46875 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.drbd1226.out‚0X€6p›•‚(: Ÿ­ÅÓ˜Œƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚132.73' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.24333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0W ¨Jc–‚(:Ø  Ÿ­ÅÓ Óу disk.drbd1817.in‚0$ memory_percent‚41.23486876487732 disk.drbd1289.out‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0# proc.memory.rssMB‚ 11254.0546875 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.drbd1673.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1931.out‚0 disk.rbd1.in‚0 disk.sda.out‚ 78.66015625 disk.drbd1510.out‚0 disk.drbd1938.out‚0 disk.rbd0.out‚0* proc.memory.percent‚17.172324657440186) disk./.usagePercent‚32.79192249856091$ disk./.usageGB‚586.3528785705566 disk.drbd1828.out‚0 disk.drbd1552.in‚0 disk.drbd1817.out‚0" disk.nvme0n1.out‚ 617.771484375 disk.drbd1289.in‚0 disk.drbd2078.out‚0 network.sent‚ 5.237977e+06 network.recv‚376885 disk.sda1.in‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.drbd1550.in‚0* proc.memory.availableMB‚38512.31640625 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd1938.in‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0 disk.sda2.out‚ 78.66015625 disk.drbd1550.out‚0 disk.sda1.out‚0 disk.sda2.in‚ 0.68359375 disk.rbd1.out‚ 9.623046875 disk.drbd1552.out‚0 disk.drbd1584.out‚0 proc.cpu.threads‚29 disk.drbd1931.in‚0 disk.sda.in‚ 0.68359375 cpu‚13.795958951557251 disk.dm-0.in‚2.46875 disk.drbd2118.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.dm-0.out‚ 617.771484375’ u¨2—- _wandbruntime‚2387  _runtime‚2387‹¦É$Ș ' lr-AdamW/pg1‚0.00014558708145549415 ' lr-AdamW/pg2‚0.00014558708145549415 trainer/global_step‚5649 $ _timestamp‚1.785812642059891e+09 _runtime‚2387.249833312 _step‚448~¾’ ꪂ _runtime‚2387.249833312 _step‚448 ' lr-AdamW/pg1‚0.00014558708145549415 ' lr-AdamW/pg2‚0.00014558708145549415 trainer/global_step‚5649 $ _timestamp‚1.785812642059891e+09ÀÔ9Ö2š- _wandbruntime‚2387  _runtime‚2387»ï1꛾O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚101968000759808 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16180502 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16180234 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚102331909412864 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.largþ+ LÚ%e_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37537527 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4971247084544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4971247084544 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37537527 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38426949 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59670531 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚96996736635904 _runtime‚2387.254631123 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59670531 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59660875 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4958993252824 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚97208060 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101967896290816 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚99545511538296 % _timestamp‚1.7858126420607946e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚101968000759808 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚96996753675264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37537529 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 _step‚449 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚96996753675264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38426679 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59660875 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚99545614380780 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚94586518285472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4959079055948 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4971159654912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4971159654912 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚96996736635904 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚01ÿ!2É'œÃO PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59670531 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59660875 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4958993252824 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚97208060 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101967896290816 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚99545511538296 % _timestamp‚1.7858126420607946e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚101968000759808 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚96996753675264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37537529 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 _step‚449 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚96996753675264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38426679 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59660875 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚99545614380780 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚94586518285472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4959079055948 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4971159654912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4971159654912 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚96996736635904 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚101968000759808 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16180502 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16180234 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚102331909412864 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37537527 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4971247084544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4971247084544 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37537527 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38426949 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59670531 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚96996736635904 _runtime‚2387.254631123ÁëSÌ2- _wandbruntime‚2387  _runtime‚2387wƒ¾Î&žÈM FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 % _timestamp‚1.7858126424093657e+09 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4959956529282 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59671398 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4972039274496 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16183332 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚97243266275328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚101986047149568 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59671398 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5106624274432 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚99563129183838 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚94603275497152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37544172 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97013920438272 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚239 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚101986047149568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22250382 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59681068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚101985942673408 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4959870726046 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚5649 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚2387.599667062 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4972039274496 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4972126711296 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16183093 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97225240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚102349865646080 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97225240 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38433716 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22250384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚94603258457792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚450 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚241 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59681068 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5106748649984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚97243241371648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37544170 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38433475 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4972126711296 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0š{ø*: ŸÍM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59671398 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4972039274496 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16183332 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚97243266275328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚101986047149568 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59671398 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5106624274432 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚99563129183838 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚94603275497152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37544172 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97013920438272 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚239 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚101986047149568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22250382 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552Ù€XM™ JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59681068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚101985942673408 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4959870726046 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚5649 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚2387.599667062 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4972039274496 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4972126711296 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16183093 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97225240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚102349865646080 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97225240 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38433716 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22250384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚94603258457792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚450 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚241 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59681068 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5106748649984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚97243241371648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37544170 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38433475 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4972126711296 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 % _timestamp‚1.7858126424093657e+09 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4959956529282¯ƒ®À2 - _wandbruntime‚2387  _runtime‚2387*() ¡ 0loss/FRt2/assignment_loss‚3.5983655910159267 1loss/SRqqt2/assignment_loss‚1.553873302255358 1loss/FBt2/assignment_loss‚0.13822424019397755 /loss/FRt2/detection_loss‚2.4432488769985907 0loss/SRqqt1/detection_loss‚0.894286023062934 /loss/FBt1/detection_loss‚3.0117525127198963 epoch‚0 _step‚451 &loss/total_loss‚22.530020274548352 trainer/global_step‚5649 % _timestamp‚1.7858126424106162e+09 0loss/FRt1/assignment_loss‚3.9259588062763213 2loss/FBt1/assignment_loss‚0.042311303595751165 /loss/FRt1/detection_loss‚2.5031718708574773 _runtime‚2387.600389023 2loss/SRqqt1/assignment_loss‚1.2164553422329387 1loss/SRqqt2/detection_loss‚2.0640207209757397 /loss/FBt2/detection_loss‚1.1383516843634425E©Ÿó ¢† 0loss/FRt2/assignment_loss‚3.5983655910159267 1loss/SRqqt2/assignment_loss‚1.553873302255358 1loss/FBt2/assignment_loss‚0.13822424019397755 /loss/FRt2/detection_loss‚2.4432488769985907 0loss/SRqqt1/detection_loss‚0.894286023062934 /loss/FBt1/detection_loss‚3.0117525127198963 epoch‚0 _step‚451 &loss/total_loss‚22.530020274548352 trainer/global_step‚5649 % _timestamp‚1.7858126424106162e+09 0loss/FRt1/assignment_loss‚3.9259588062763213 2loss/FBt1/assignment_loss‚0.042311303595751165 /loss/FRt1/detection_loss‚2.5031718708574773 _runtime‚2387.600389023 2loss/SRqqt1/assignment_loss‚1.2164553422329387 1loss/SRqqt2/detection_loss‚2.0640207209757397 /loss/FBt2/detection_loss‚1.1383516843634425ÃX¸œ£‚(:‘ ®­ÅӲШƒ gpu.0.gpu‚44.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚132.979' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.32633333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0S¨íþ\¤‚(:Ñ  ®­ÅÓÖí΃ disk.drbd2078.out‚0 disk.drbd1938.in‚0 disk.drbd1285.out‚0* disk./.usagePercent‚32.791923565250116 disk.rbd1.out‚ 9.685546875 cpu‚13.795121051346813 disk.nvme0n1.in‚2.46875 disk.sda.in‚ 0.68359375 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1226.out‚0 network.recv‚377825 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.drbd2111.in‚0 disk.drbd1289.in‚0 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.drbd1289.out‚0 proc.cpu.threads‚29 disk.drbd2041.out‚0 disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0* proc.memory.percent‚17.172324657440186# disk./.usageGB‚586.352897644043 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.drbd1828.in‚0 disk.dm-0.out‚ 618.13671875 disk.drbd2118.out‚0 network.sent‚ 5.24105e+06 disk.drbd1817.out‚0 disk.drbd1673.in‚0# proc.memory.rssMB‚ 11254.0546875! disk.nvme0n1.out‚ 618.13671875 disk.dm-0.in‚2.46875 disk.sda1.in‚0 disk.sda2.out‚ 79.015625 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.drbd1673.out‚0$ memory_percent‚41.23496413230896* proc.memory.availableMB‚38512.25390625 disk.sda.out‚ 79.015625 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.drbd1965.in‚0ˆ±¸’2¥- _wandbruntime‚2407  _runtime‚2407”§É¦Ã ' lr-AdamW/pg2‚0.00014687569077975944 trainer/global_step‚5699 % _timestamp‚1.7858126627932022e+09 _runtime‚2407.983147041 _step‚452 ' lr-AdamW/pg1‚0.00014687569077975944ešñÎ§È ' lr-AdamW/pg1‚0.00014687569077975944 ' lr-AdamW/pg2‚0.00014687569077975944 trainer/global_step‚5699 % _timestamp‚1.7858126627932022e+09 _runtime‚2407.983147041 _step‚452Ä Eôì2¨- _wandbruntime‚2407  _runtime‚2407’ŽôUÆ'©ÀO ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 trainer/global_step‚5699 _runtime‚2407.987648574 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5151050587136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚242 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚103237310007808 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚95423526901472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚102870335045632 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚102870230576640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚97855074786304 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚102870230576640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16324117 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 187192320 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37869677 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37869679 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22443295 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚60188877 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚60188877 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚102870335045632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5015243219968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚103237465585664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60198533 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5015243219968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98058554 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚100426511460288 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _step‚453 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37869679 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚97855091825664 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚102870335045632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5015243219968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚103237465585664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60198533 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5015243219968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98058554 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚100426511460288 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _step‚453 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37869679 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚97855091825664 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚102870230576640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚97855074786304 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2ÅÎ_MJ2«- _wandbruntime‚2408  _runtime‚2408›,pA ¬ÊM E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98085392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37876322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22447234 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98103597211648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5151819765760 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60199400 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚95440284113152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16326690 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37876320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚98103622115328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5003844992790 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37876322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5016122846720 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22447232 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚100444129105942 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97872258588672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5003759189554 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚100444026263346 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚102888276959232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5016035409920 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38774184 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38773922 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5151954627072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚103255576742400 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37876320 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚103255416977408 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚102888381435392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚102888381435392 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 % _timestamp‚1.7858126631458101e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98085392 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98075720 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚95440267073792ušNÕ&­ÏM \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚103255576742400 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37876320 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚103255416977408 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚102888381435392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚102888381435392 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 % _timestamp‚1.7858126631458101e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98085392 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98075720 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚95440267073792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98085392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37876322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22447234 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98103597211648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5151819765760 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60199400 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚95440284113152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16326690 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37876320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚98103622115328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5003844992790 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37876322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5016122846720 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22447232 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚100444129105942 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97872258588672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5003759189554 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚100444026263346 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚102888276959232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5016035409920 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38774184 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38773922 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5151954627072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136ÆRÆæ•2®- _wandbruntime‚2408  _runtime‚2408jAa¯ý 1loss/FBt2/assignment_loss‚0.06002205052438699 .loss/FRt1/detection_loss‚2.604151370300763 .loss/FRt2/detection_loss‚2.637636053775038 0loss/SRqqt2/detection_loss‚2.367256625340535 epoch‚0 trainer/global_step‚5699 _runtime‚2408.337120904 /loss/FRt1/assignment_loss‚3.526555973550548 2loss/SRqqt2/assignment_loss‚1.5454081064806535 /loss/FBt2/detection_loss‚1.4072467562831827 %loss/total_loss‚36.40019978086025 _step‚455 2loss/SRqqt1/assignment_loss‚1.3453500064282582 1loss/FBt1/assignment_loss‚0.14485374084536792 1loss/SRqqt1/detection_loss‚0.8258046617745518 /loss/FBt1/detection_loss‚16.623636960983276 % _timestamp‚1.7858126631470218e+09 0loss/FRt2/assignment_loss‚3.3122774745736803™¥÷°‚ trainer/global_step‚5699 _runtime‚2408.337120904 /loss/FRt1/assignment_loss‚3.526555973550548 2loss/SRqqt2/assignment_loss‚1.5454081064806535 /loss/FBt2/detection_loss‚1.4072467562831827 %loss/total_loss‚36.40019978086025 _step‚455 2loss/SRqqt1/assignment_loss‚1.3453500064282582 1loss/FBt1/assignment_loss‚0.14485374084536792 1loss/SRqqt1/detection_loss‚0.8258046617745518 /loss/FBt1/detection_loss‚16.623636960983276 % _timestamp‚1.7858126631470218e+09 0loss/FRt2/assignment_loss‚3.3122774745736803 1loss/FBt2/assignment_loss‚0.06002205052438699 .loss/FRt1/detection_loss‚2.604151370300763 .loss/FRt2/detection_loss‚2.637636053775038 0loss/SRqqt2/detection_loss‚2.367256625340535 epoch‚0Ç=Ѯ𜱂(:‘ ½­ÅÓé稃 gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚106.582' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚35.52733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0æòac²‚(:Ø  ½­ÅÓò̶ƒ disk.rbd0.in‚0 disk.drbd2111.out‚0 disk.drbd1510.out‚0 disk.rbd1.in‚0$ disk./.usageGB‚586.3528785705566 disk.drbd1289.in‚0 disk.sda2.out‚ 79.3359375 disk.drbd1226.in‚0 disk.drbd1285.in‚0 disk.dm-0.out‚618.5556640625 disk.drbd2118.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd2078.out‚0* proc.memory.percent‚17.172324657440186 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.sda.in‚ 0.68359375 disk.dm-0.in‚2.46875 disk.drbd1552.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 network.recv‚380293 disk.drbd1356.in‚0 disk.drbd1550.out‚0 proc.cpu.threads‚29 disk.rbd0.out‚0 disk.drbd1510.in‚0$ memory_percent‚41.23501181602478 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.drbd1931.in‚0 disk.drbd1938.out‚0 disk.drbd1938.in‚0 network.sent‚ 5.286032e+06 disk.sda.out‚ 79.3359375 disk.sda1.in‚0 disk.drbd1552.out‚0# disk.nvme0n1.out‚618.5556640625# proc.memory.rssMB‚ 11254.0546875 disk.drbd1356.out‚0 disk.drbd1673.in‚0 cpu‚13.794295257480016) disk./.usagePercent‚32.79192249856091 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd2041.in‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.sda2.in‚ 0.68359375 disk.rbd1.out‚ 9.685546875* proc.memory.availableMB‚38512.22265625 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0Rç¥2³- _wandbruntime‚2428  _runtime‚2428ÄòÀµÈ´Â ' lr-AdamW/pg1‚0.00014816430010402475 ' lr-AdamW/pg2‚0.00014816430010402475 trainer/global_step‚5749 % _timestamp‚1.7858126834583988e+09 _runtime‚ 2428.64834407 _step‚456Ý¢“ƒÍµÇ _runtime‚ 2428.64834407 _step‚456 ' lr-AdamW/pg1‚0.00014816430010402475 ' lr-AdamW/pg2‚0.00014816430010402475 trainer/global_step‚5749 % _timestamp‚1.7858126834583988e+09ÈÎLz2¶- _wandbruntime‚2428  _runtime‚2428þ?XxÜ%·¿O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5059239355392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38201827 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚2428.652629949 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚103772564862464 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚98928364 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38201829 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚60716879 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39107666 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚103772564862464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5046855982964 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _step‚457 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39107404 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16467259 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22640147 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚98713412936704 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚96260552556832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98918706 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 I¶ éDeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 $ _timestamp‚1.785812683459335e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130—úe¡Ê'¸ÄO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚60716879 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39107666 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚103772564862464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5046855982964 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _step‚457 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39107404 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16467259 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22640147 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚98713412936704 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚96260552556832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98918706 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 _runtime‚2428.652629949 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚103772564862464 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚98928364 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38201829 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484Éû³¡Ó2¹- _wandbruntime‚2429  _runtime‚2429W$ÉÏ&ºÉM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98945544 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38208472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5060031545344 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38208472 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚103790611245056 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚98730579699712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚96277292729152 _step‚458 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚98963977955328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚104161088712704 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5196973798912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚98730596739072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚101325026185450 trainer/global_step‚5749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚104160926850560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚103790715721216 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98963953051648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚98730579699712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22644084 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚2429.000253126 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98945544 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5060118982144 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60737072 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 $ _timestamp‚1.785812683809979e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5047647653062 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38208470 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39114181 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚96277275689792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚103790715721216 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚98730596739072 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5060031545344 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5047733456298 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60727402 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98935872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60737072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60727402 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130ƒ¥ÁÔ&»ÎM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38208470 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39114181 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚96277275689792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚103790715721216 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚98730596739072 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5060031545344 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5047733456298 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60727402 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98935872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60737072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60727402 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98945544 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38208472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5060031545344 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38208472 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚103790611245056 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚98730579699712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚96277292729152 _step‚458 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚98963977955328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚104161088712704 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5196973798912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚98730596739072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚101325026185450 trainer/global_step‚5749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚104160926850560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚103790715721216 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98963953051648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚98730579699712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22644084 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚2429.000253126 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98945544 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5060118982144 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60737072 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 $ _timestamp‚1.785812683809979e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5047647653062Êg* 2¼- _wandbruntime‚2429  _runtime‚2429ÜÅO’ÿ½ù 2loss/SRqqt1/assignment_loss‚1.3975982749757687 0loss/FBt1/assignment_loss‚0.1107337658779995 1loss/SRqqt2/detection_loss‚2.0909036147727895 _step‚459 /loss/FRt1/assignment_loss‚4.079160109643014 .loss/FBt2/detection_loss‚0.995261200144887 &loss/total_loss‚21.486165267364527 .loss/FRt2/assignment_loss‚3.42398519217968 1loss/SRqqt2/assignment_loss‚1.438780392283824 /loss/FRt1/detection_loss‚2.3538409208738678 -loss/FRt2/detection_loss‚2.54827910810709 .loss/FBt1/detection_loss‚1.882551638143403 epoch‚0 % _timestamp‚1.7858126838111813e+09 1loss/FBt2/assignment_loss‚0.09777232574326718 1loss/SRqqt1/detection_loss‚1.0672987246189383 trainer/global_step‚5749 _runtime‚2429.001147227Ø“o;¾þ /loss/FRt1/assignment_loss‚4.079160109643014 .loss/FBt2/detection_loss‚0.995261200144887 &loss/total_loss‚21.486165267364527 .loss/FRt2/assignment_loss‚3.42398519217968 1loss/SRqqt2/assignment_loss‚1.438780392283824 /loss/FRt1/detection_loss‚2.3538409208738678 -loss/FRt2/detection_loss‚2.54827910810709 .loss/FBt1/detection_loss‚1.882551638143403 epoch‚0 % _timestamp‚1.7858126838111813e+09 1loss/FBt2/assignment_loss‚0.09777232574326718 1loss/SRqqt1/detection_loss‚1.0672987246189383 trainer/global_step‚5749 _runtime‚2429.001147227 2loss/SRqqt1/assignment_loss‚1.3975982749757687 0loss/FBt1/assignment_loss‚0.1107337658779995 1loss/SRqqt2/detection_loss‚2.0909036147727895 _step‚459ËÓšÈt¿‚(:Ô  Ì­ÅÓªþÔƒ network.recv‚382739# disk./.usageGB‚586.352840423584 disk.drbd1965.out‚0 disk.rbd0.in‚0 disk.drbd1938.out‚0* proc.memory.percent‚17.172324657440186 disk.sda.out‚ 79.7265625 disk.drbd2078.in‚0 disk.drbd1289.out‚0# proc.memory.rssMB‚ 11254.0546875$ disk.nvme0n1.out‚619.17919921875& proc.memory.availableMB‚ 38512.4375 disk.rbd1.in‚0 disk.rbd1.out‚ 9.833984375 disk.drbd1226.in‚0 disk.drbd1828.out‚0 disk.drbd1817.out‚IK n[0 disk.drbd1226.out‚0 disk.drbd1828.in‚0 disk.drbd2118.out‚0 disk.drbd1356.out‚0 disk.sda.in‚ 0.68359375 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 cpu‚13.793925720535542 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd2078.out‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd1289.in‚0 network.sent‚ 5.33117e+06 disk.drbd1584.in‚0 disk.drbd1285.in‚0! disk.dm-0.out‚619.17919921875 disk.drbd1817.in‚0 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0( disk./.usagePercent‚32.7919203651825 disk.drbd1931.out‚0 disk.drbd1673.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.drbd1584.out‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd2111.in‚0% memory_percent‚41.234683990478516 disk.drbd1356.in‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚2.46875 disk.dm-0.in‚2.46875 disk.drbd1510.out‚0 proc.cpu.threads‚29 disk.sda2.out‚ 79.7265625H±À‚(:’ Ì­ÅÓŽ²Ðƒ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚132.398' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.132666666666665 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0³riÁ‚(:’ Û­ÅÓð°ü‚ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚107.997' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚35.998999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 Ÿ^‚(:Ó  Û­ÅÓ®£Âƒ$ disk./.usageGB‚586.3530464172363 disk.drbd1965.in‚0 proc.cpu.threads‚29) disk./.usagePercent‚32.79193188542591 disk.sda1.in‚0 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.drbd2078.out‚0 disk.drbd1817.in‚0 disk.sda2.in‚ 0.68359375 disk.rbd1.in‚0# proc.memory.rssMB‚ 11254.0546875 disk.drbd2118.in‚0 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1965.out‚0 disk.sda.in‚ 0.68359375 disk.drbd2118.out‚0 disk.sda1.out‚0 disk.drbd1673.in‚0 disk.drbd1938.out‚0 disk.rbd1.out‚ 9.927734375 disk.dm-0.out‚ 619.71484375 network.recv‚385127 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd1584.out‚0 disk.sda.out‚ 80.03515625 disk.drbd1510.out‚0 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.dm-0.in‚2.46875 disk.drbd1289.out‚0! disk.nvme0n1.out‚ 619.71484375 disk.drbd1584.in‚0 disk.rbd0.in‚0 cpu‚13.79311646559913 disk.drbd1356.in‚0 network.sent‚ 5.37615e+06 disk.drbd1289.in‚0 disk.drbd2111.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd2041.in‚0 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.drbd1285.in‚0) proc.memory.availableMB‚ 38512.1484375 disk.drbd1510.in‚0 disk.drbd1552.in‚0 disk.sda2.out‚ 80.03515625* proc.memory.percent‚17.172324657440186 disk.drbd1285.out‚0 disk.drbd1828.in‚0$ memory_percent‚41.23512506484985 disk.drbd1552.out‚0 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1817.out‚0 disk.drbd1356.out‚03==j2Ã- _wandbruntime‚2449  _runtime‚2449k‡»òÈÄ ' lr-AdamW/pg1‚0.00014945290942829006 ' lr-AdamW/pg2‚0.00014945290942829006 trainer/global_step‚5799 $ _timestamp‚1.785812704251886e+09 _runtime‚2449.441840105 _step‚460ÕdšÍÅÇ _step‚460 ' lr-AdamW/pg1‚0.00014945290942829006 ' lr-AdamW/pg2‚0.00014945290942829006 trainer/global_step‚5799 $ _timestamp‚1.785812704251886e+09 _runtime‚2449.441840105̯¤–µ2Æ- _wandbruntime‚2449  _runtime‚2449˽×ËÆ'ÇÀO cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5241383591424 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚99571768126464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38533979 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚99788516 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39447620 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16610625 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚102188305619304 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5103235490816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39447874 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5241250820096 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚97097544133472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚99807101774848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚105048352594944 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38533979 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5103148061184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22836995 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚105048510269952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚461 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚104674899148288 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚99807126678528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚102188202776820 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 % _timestamp‚1.7858127042528653e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚99778858 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _runtime‚2449.446681636 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61244881 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚104675003617280 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚99571751087104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38533977 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚104675003617280 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚104674899148288 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38533977 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61244881 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5090744446472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚105048352594944 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38533979 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5103148061184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22836995 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚105048510269952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚461 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚104674899148288 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚99807126678528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚102188202776820 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 % _timestamp‚1.7858127042528653e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚99778858 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _runtime‚2449.446681636 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61244881 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚104675003617280 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚99571751087104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38533977 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚104675003617280 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚104674899148288 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38533977 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61244881 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5090744446472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5104115117568 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5242154498560 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22840932 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚5799 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚104692945530880 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39454298 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99796024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22840934 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61255404 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99805696 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38540620 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚99824308891648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚97114284305792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚99588917850112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99805696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚105066621058048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 _step‚462 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5104115117568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38540622 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5091621919806 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚104693050007040 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceË©¼wÓStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚99588934889472 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚102205820422362 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61255404 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5091536116570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39454298 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99796024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22840934 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61255404 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99805696 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38540620 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚99824308891648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚97114284305792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚99588917850112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99805696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚105066621058048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 _step‚462 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5104115117568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38540622 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5091621919806 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚104693050007040 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚99588934889472 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚102205820422362 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61255404 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5091536116570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5104115117568 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5242154498560 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22840932 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚5799 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚104692945530880 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38866127 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 100639010 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚100430106276864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61772883 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚103069099856328 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5147231626240 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 % _timestamp‚1.7858127250944767e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚105577337903104 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚105577233434112 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚97934569788832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚2470.284797805 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5147231626240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38866129 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5147144196608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚105577337903104 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39786335 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚100667457614848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _step‚465 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61772883 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5147144196608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38866129 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚100430106276864 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5286422642688 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5134632909980 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105577233434112 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100648668 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚105954031641088 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61782539 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5286549122560 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚97934552749472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23033845 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16752251 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130?çIðÖÏO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38866127 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 100639010 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚100430106276864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61772883 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚103069099856328 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5147231626240 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 % _timestamp‚1.7858127250944767e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚105577337903104 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚105577233434112 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚97934569788832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚2470.284797805 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5147231626240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38866129 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5147144196608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚105577337903104 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39786335 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚100667457614848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _step‚465 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61772883 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5147144196608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38866129 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚100430106276864 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5286422642688 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5134632909980 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105577233434112 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100648668 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚105954031641088 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61782539 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5286549122560 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚97934552749472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool•»"å .peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23033845 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16752251 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚5849 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5134547106856ѱÓ"s2×- _wandbruntime‚2470  _runtime‚2470pDÔÚ&ØÔM JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38872772 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚97951309961152 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 100656176 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61783406 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚105595384292864 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚97951292921792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚105595384292864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858127254410172e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100665848 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚105595279816704 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5148111252992 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100665848 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61793076 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5148023816192 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5135424580078 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5148111252992 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39792774 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚103086717501870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚100684689635328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚105972145943552 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚105971992470016 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚103086820344466 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16754992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 trainer/global_step‚5849 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61793076 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚100447256000512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100656176 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38872770 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38872772 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23037782 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5287327738368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39793036 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23037784 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _runtime‚2470.631277645 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61783406v†ß&ÙÙM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858127254410172e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100665848 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚105595279816704 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5148111252992 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100665848 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61793076 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5148023816192 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5135424580078 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5148111252992 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39792774 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚103086717501870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚100684689635328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚105972145943552 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚105971992470016 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚103086820344466 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16754992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 trainer/global_step‚5849 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61793076 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚100447256000512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100656176 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38872770 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38872772 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23037782 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5287327738368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39793036 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23037784 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _runtime‚2470.631277645 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61783406 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38872772 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚97951309961152 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 100656176 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61783406 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚105595384292864 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚97951292921792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚105595384292864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0Ò3'n 2Ú- _wandbruntime‚2470  _runtime‚2470ÞQÐAÛ€ 1loss/SRqqt2/detection_loss‚2.1325435054798922 /loss/FBt2/detection_loss‚1.1979875006895913 &loss/total_loss‚21.796593602608212 epoch‚0 _step‚467 2loss/SRqqt1/assignment_loss‚1.2455528264908935 /loss/FRt1/detection_loss‚2.4150358246248445 1loss/SRqqt1/detection_loss‚0.9321722077155243 trainer/global_step‚5849 _runtime‚2470.631989164 1loss/FBt1/assignment_loss‚0.40449649765650975 /loss/FRt2/assignment_loss‚3.643815906568505 2loss/SRqqt2/assignment_loss‚1.1922273163994153 /loss/FRt2/detection_loss‚2.4023251879489287 /loss/FBt1/detection_loss‚3.0218102236588797 % _timestamp‚1.7858127254422047e+09 /loss/FRt1/assignment_loss‚3.121539561100948 1loss/FBt2/assignment_loss‚0.08708704427428009È2µ Ü… /loss/FRt1/assignment_loss‚3.121539561100948 1loss/FBt2/assignment_loss‚0.08708704427428009 1loss/SRqqt2/detection_loss‚2.1325435054798922 /loss/FBt2/detection_loss‚1.1979875006895913 &loss/total_loss‚21.796593602608212 epoch‚0 _step‚467 2loss/SRqqt1/assignment_loss‚1.2455528264908935 /loss/FRt1/detection_loss‚2.4150358246248445 1loss/SRqqt1/detection_loss‚0.9321722077155243 trainer/global_step‚5849 _runtime‚2470.631989164 1loss/FBt1/assignment_loss‚0.40449649765650975 /loss/FRt2/assignment_loss‚3.643815906568505 2loss/SRqqt2/assignment_loss‚1.1922273163994153 /loss/FRt2/detection_loss‚2.4023251879489287 /loss/FBt1/detection_loss‚3.0218102236588797 % _timestamp‚1.7858127254422047e+09Ó†0ôœÝ‚(:‘ ù­ÅÓ¾¢ƒ gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚134.144' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.714666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$͈²cÞ‚(:Ø  ù­ÅÓ®ÚɃ$ disk./.usageGB‚586.3553771972656 disk.drbd1552.out‚0 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.drbd2111.out‚0$ proc.memory.rssMB‚11254.05859375& proc.memory.availableMB‚ 38512.3125 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.sda1.in‚0 disk.drbd2118.out‚0$ disk.nvme0n1.out‚620.75830078125 disk.drbd1510.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.drbd2041.in‚0 disk.sda.out‚ 81.08984375 disk.drbd1828.out‚0 disk.drbd1552.in‚0 disk.drbd1938.out‚0 disk.dm-0.in‚2.46875 cpu‚13.791719662040988 disk.drbd1356.in‚0 proc.cpu.threads‚29 disk.drbd1584.in‚0) disk./.usagePercent‚32.79206223484666 disk.sda.in‚ 0.68359375* proc.memory.percent‚17.172330617904663 disk.drbd1673.out‚0 disk.sda2.out‚ 81.08984375 disk.nvme0n1.in‚2.46875 disk.drbd1828.in‚0 disk.drbd1931.in‚0 disk.rbd0.out‚0 network.sent‚ 5.424219e+06 network.recv‚388600 disk.sda1.out‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd2078.out‚0# memory_percent‚41.2348747253418 disk.rbd1.out‚ 9.990234375 disk.drbd2078.in‚0 disk.drbd1510.in‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0 disk.drbd1584.out‚0 disk.drbd1550.in‚0! disk.dm-0.out‚620.75830078125 disk.drbd2111.in‚0Nyç¸eß‚(:Ú  ˆ®ÅÓ—–݃$ disk./.usageGB‚586.3553771972656 disk.drbd2118.in‚0 disk.drbd1584.in‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.drbd1817.in‚0 disk.drbd1550.in‚0 disk.drbd1817.out‚0 proc.cpu.threads‚29* proc.memory.availableMB‚38512.32421875 network.sent‚ 5.469299e+06) disk./.usagePercent‚32.79206223484666 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.sda2.out‚ 81.484375 disk.rbd1.out‚ 10.146484375 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.sda1.in‚0 cpu‚13.79098328104788 disk.sda.out‚ 81.484375 disk.drbd1938.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1965.out‚0 disk.drbd1510.out‚0 disk.drbd1931.in‚0 disk.drbd1356.out‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1226.out‚0 disk.drbd1828.in‚0 disk.drbd1931.out‚0 disk.nvme0n1.in‚2.46875 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1356.in‚0! disk.dm-0.out‚621.05517578125 disk.drbd1552.in‚0 disk.sda2.in‚ 0.68359375* proc.memory.percent‚17.172330617904663$ disk.nvme0n1.out‚621.05517578125 disk.rbd0.out‚0 disk.drbd1285.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2041.out‚0 disk.dm-0.in‚2.46875 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd1828.out‚0 disk.drbd2118.out‚0 network.recv‚391016 disk.drbd2041.in‚0 disk.drbd1552.out‚0% memory_percent‚41.2348568439483644M›à‚(: ˆ®ÅÓéšÛƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚130.93' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.64333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0quË2á- _wandbruntime‚2491  _runtime‚2491\ AdÉâà _step‚468 ' lr-AdamW/pg1‚0.00015203012807682068 ' lr-AdamW/pg2‚0.00015203012807682068 trainer/global_step‚5899 % _timestamp‚1.7858127458276935e+09 _runtime‚2491.017697918Ò‡×ÎãÈ ' lr-AdamW/pg1‚0.00015203012807682068 ' lr-AdamW/pg2‚0.00015203012807682068 trainer/global_step‚5899 % _timestamp‚1.7858127458276935e+09 _runtime‚2491.017697918 _step‚468Ôíì&2ä- _wandbruntime‚2491  _runtime‚2491Ùý4ƒ åÊO ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚106859598652928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _runtime‚2491.022635499 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40125080 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5331760294400 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62300885 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5191140332032 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚98771578404832 % _timestamp‚1.7858127458286982e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5331629620224 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16894385 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 _step‚469 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚106479672188928 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5191227761664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39198277 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚103950099778320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚106479567719936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23230695 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚106859443075072 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39198277 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚101288427387904 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 101508820 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚244 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚101527838358528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5191140332032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚101288444427264 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚101288427387904 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62310541 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39198279 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚242 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39198279 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚62300885 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5178521373488 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚106479672188928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16894627 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5191227761664 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚101527813454848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40125324 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚101288444427264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23230697 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5178521373488 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚106479672188928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16894627 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5191227761664 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚101527813454848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40125324 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚101288444427264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23230697 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚106479567719936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23230695 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚106859443075072 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39198277 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚101288427387904 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 101508820 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚244 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚101527838358528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5191140332032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚101288444427264 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚101288427387904 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62310541Õ™›¿^2ç- _wandbruntime‚2491  _runtime‚2491‰KYÚ&èÔM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚106497718578688 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚62311408 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5332531217920 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5179313043586 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39204922 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39204920 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚5899 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16897227 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚101545020571648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101526000 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚103967717423974 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚101305611190272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5192107388416 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚98788318577152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23234632 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5192107388416 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5192019951616 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40131859 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5179398846822 _step‚470 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚98788301537792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚103967614581378 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101516328 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23234634 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚106497718578688 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101526000 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5332666079232 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚2491.370618466 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚106877551789568 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40132115 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39204922 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858127461803086e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚101305611190272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 101516328 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858127461803086e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚101305611190272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 101516328 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39204922 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39204920 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚5899 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16897227 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚101545020571648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101526000 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚103967717423974 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚101305611190272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5192107388416 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚98788318577152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23234632 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5192107388416 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5192019951616 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40131859 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5179398846822 _step‚470 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚98788301537792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚103967614581378 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101516328 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23234634 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚106497718578688 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101526000 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5332666079232 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚2491.370618466 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚106877551789568 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40132115 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39204922 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚107381902005760 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5222409836996 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5235223897088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17035967 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5376762848256 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚107382006474752 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚102146765538304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23427547 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5235136467456 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 102368972 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39530427 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40463512 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚107764932143104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚104830996857828 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚99608587020832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102359314 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39530429 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39530427 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62828887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39530429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23427545 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚102146782577664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚5949 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚102388169294848±ù=²Õ'ôÏO NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚104830996857828 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚99608587020832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102359314 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39530429 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39530427 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62828887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39530429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23427545 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚102146782577664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚5949 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚102388169294848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚107381902005760 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5235223897088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚107765087720960 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚107382006474752 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62838543 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5376893522432 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚277 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚102146782577664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5235136467456 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40463789 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚275 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚99608569981472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚2511.835629468 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62838543 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17036242 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚107381902005760 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5222409836996 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5235223897088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17035967 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5376762848256 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚107382006474752 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚102146765538304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23427547 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5235136467456 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 102368972 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39530427 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40463512 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚107764932143104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144ÙzŽ Î2õ- _wandbruntime‚2512  _runtime‚2512~´¢Fh öÔM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5223201507094 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5223287310330 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102386152 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858127669940717e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62839410 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17039057 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40470285 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 102376480 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚102405401315328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5377795383296 Dju¼’r>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102386152 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39537072 _step‚474 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚102405376411648 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚107400052864512 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚102163932301312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 trainer/global_step‚5949 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62849080 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚2512.184336028 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17038803 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5236103523840 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚107399948388352 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39537070 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5236103523840 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚99625327193152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62849080 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚107400052864512 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚107783047419392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39537072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚104848614503482 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102386152 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858127669940717e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62839410 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17039057 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40470285 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 102376480 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚102405401315328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5377795383296 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102386152 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39537072 _step‚474 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚102405376411648 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚107400052864512 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚102163932301312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 trainer/global_step‚5949 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62849080 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚2512.184336028 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17038803 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5236103523840 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚107399948388352 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39537070 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5236103523840 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚99625327193152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62849080 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚107400052864512 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚107783047419392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39537072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚104848614503482 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39862577 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63356889 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚103005103688704 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚108284236291584 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40804175 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5279132602880 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23624397 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚103248550038528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5279220032512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 103229124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17179533 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39862579 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚100445595636832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5422099227136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚103005103688704 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚108284236291584 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5279220032512 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚108284340760576 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40803928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5421966455808 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5266212497380 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚105711791094852 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63366545 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39862579 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63356889 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39862577 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚108670649265664 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858127874361603e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 103219466 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5279132602880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚103248525134848 _step‚477 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚108670491590656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚103005120728064 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚100445578597472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23624395 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5266298300504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39862577 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63356889 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚103005103688704 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚108284236291584 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40804175 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5279132602880 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23624397 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚103248550038528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5279220032512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 103229124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17179533 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39862579 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚100445595636832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5422099227136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚103005103688704 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚108284236291584 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5279220032512 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚108284340760576 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40803928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5421966455808 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5266212497380 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚105711791094852 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63366545 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39862579 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63356889 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39862577 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚108670649265664 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858127874361603e+09Ýk=ª2ƒ- _wandbruntime‚2532  _runtime‚2532 ˆ©Ü&„ÖM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17182581 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5280012222464 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40810915 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚2532.976692233 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 103236632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚103265757155328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39869222 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23628334 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108688760803328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103246304 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚108302387150336 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚103022287491072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103246304 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚108688603135488 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5280012222464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚100462335809152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40810644 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63377082 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63367412 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚108302387150336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5280099659264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5423003648000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚105729408740394 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103022270451712 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚103265732251648 _step‚478 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚103022270451712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚271 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚105729511582990 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23628332 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚5999 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17182312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5422870883840 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63367412 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39869222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚103022287491072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚269 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858127877863834e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39869220 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5280099659264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5267089970602 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251q×.á&…ÛM REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17182312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5422870883840 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63367412 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39869222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚103022287491072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚269 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858127877863834e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39869220 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5280099659264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5267089970602 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17182581 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5280012222464 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40810915 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚2532.976692233 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 103236632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚103265757155328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39869222 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23628334 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108688760803328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103246304 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚108302387150336 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚103022287491072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103246304 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚108688603135488 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5280012222464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚100462335809152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40810644 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63377082 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63367412 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚108302387150336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5280099659264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5423003648000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚105729408740394 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103022270451712 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚103265732251648 _step‚478 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚103022270451712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚271 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚105729511582990 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23628332 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚5999 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712Þ÷¤Ä2†- _wandbruntime‚2532  _runtime‚2532V$v‡ /loss/FBt1/detection_loss‚3.9930972903966904 /loss/FBt2/detection_loss‚0.8266114179904644 trainer/global_step‚5999 _runtime‚2532.977419342 /loss/FRt2/detection_loss‚2.7096241194950905 1loss/SRqqt1/detection_loss‚0.8804453178275403 % _timestamp‚1.7858127877876256e+09 2loss/SRqqt2/assignment_loss‚1.2686672798897092 /loss/FRt1/detection_loss‚2.6702633655782955 0loss/SRqqt2/detection_loss‚2.177874515911466 _step‚479 .loss/FRt1/assignment_loss‚3.53395404121769 0loss/FRt2/assignment_loss‚3.4250804891711786 1loss/SRqqt1/assignment_loss‚1.285969220584245 1loss/FBt2/assignment_loss‚0.05550116381193959 &loss/total_loss‚22.827255741002666 epoch‚0 4loss/FBt1/assignment_loss‚0.00016751912835388794fS4 ˆ† 4loss/FBt1/assignment_loss‚0.00016751912835388794 /loss/FBt1/detection_loss‚3.9930972903966904 /loss/FBt2/detection_loss‚0.8266114179904644 trainer/global_step‚5999 _runtime‚2532.977419342 /loss/FRt2/detection_loss‚2.7096241194950905 1loss/SRqqt1/detection_loss‚0.8804453178275403 % _timestamp‚1.7858127877876256e+09 2loss/SRqqt2/assignment_loss‚1.2686672798897092 /loss/FRt1/detection_loss‚2.6702633655782955 0loss/SRqqt2/detection_loss‚2.177874515911466 _step‚479 .loss/FRt1/assignment_loss‚3.53395404121769 0loss/FRt2/assignment_loss‚3.4250804891711786 1loss/SRqqt1/assignment_loss‚1.285969220584245 1loss/FBt2/assignment_loss‚0.05550116381193959 &loss/total_loss‚22.827255741002666 epoch‚0ßëØî÷›‰‚(: µ®ÅÓ󣟃 gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚134.261' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.75366666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ãG­\ZŠ‚(:Ï  µ®Åӛݸƒ disk.nvme0n1.out‚ 621.7890625 disk.drbd1584.in‚0 disk.drbd1965.in‚0 disk.drbd1828.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd2118.out‚0 disk.drbd1673.in‚0 disk.drbd1938.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.drbd1356.in‚0 disk.dm-0.out‚ 621.7890625 disk.drbd1931.in‚0 disk.drbd1289.in‚0 network.sent‚ 5.56242e+06 disk.sda1.in‚0 disk.nvme0n1.in‚2.46875 disk.drbd2041.in‚0 disk.drbd1584.out‚0$ memory_percent‚41.23506546020508 disk.drbd1817.out‚0 disk.drbd2078.out‚0 disk.drbd2111.out‚0) disk./.usagePercent‚32.79206116815746 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd1931.out‚0 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.drbd1510.out‚0& proc.memory.availableMB‚ 38512.1875 disk.drbd1817.in‚0 cpu‚13.788303904839333 disk.rbd1.in‚0 disk.drbd1965.out‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.drbd1356.out‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0$ disk./.usageGB‚586.3553581237793 disk.sda2.in‚ 0.68359375 disk.sda.out‚ 82.3984375 network.recv‚396598 disk.drbd2118.in‚0 disk.rbd1.out‚ 10.302734375 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.drbd1510.in‚0 disk.sda2.out‚ 82.3984375 disk.dm-0.in‚2.46875$ proc.memory.rssMB‚11254.05859375 proc.cpu.threads‚29 disk.drbd1938.in‚0‚beÐœ‹‚(:‘ Ä®ÅÓ׈›ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚133.086' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.36200000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0½ÜŽpŒ‚(:â  Ä®ÅÓÁŒÈƒ disk.sda.in‚ 0.68359375 disk.drbd1550.in‚0 disk.drbd2111.out‚0 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.drbd1285.in‚0 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.sda2.out‚ 82.83203125 disk.drbd1356.out‚0$ disk./.usageGB‚586.3553581237793 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.Ë,BÊýdrbd2118.in‚0) disk./.usagePercent‚32.79206116815746 disk.drbd1965.in‚0 disk.drbd1510.in‚0 disk.drbd1938.out‚0$ disk.nvme0n1.out‚622.07958984375* proc.memory.availableMB‚38512.18359375 network.sent‚ 5.6077e+06 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.drbd1817.out‚0 disk.rbd1.in‚0% memory_percent‚41.235071420669556 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1673.out‚0 proc.cpu.threads‚29 disk.drbd2118.out‚0 disk.drbd2041.out‚0 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.sda1.in‚0 disk.rbd1.out‚ 10.3349609375 disk.nvme0n1.in‚ 4.9296875* proc.memory.percent‚17.172330617904663 network.recv‚398819 cpu‚13.787747845136039 disk.sda2.in‚ 0.68359375 disk.dm-0.in‚ 4.9296875! disk.dm-0.out‚622.07958984375 disk.drbd1550.out‚0 disk.drbd1285.out‚0 disk.drbd1356.in‚0 disk.sda.out‚ 82.83203125 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.drbd1938.in‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11254.05859375 x2- _wandbruntime‚2553  _runtime‚25538ð΃ÇŽÁ & lr-AdamW/pg1‚0.0001558959560496166 & lr-AdamW/pg2‚0.0001558959560496166 trainer/global_step‚6049 % _timestamp‚1.7858128083595464e+09 _runtime‚2553.549488894 _step‚480Ò=€8ÌÆ % _timestamp‚1.7858128083595464e+09 _runtime‚2553.549488894 _step‚480 & lr-AdamW/pg1‚0.0001558959560496166 & lr-AdamW/pg2‚0.0001558959560496166 trainer/global_step‚6049àò?‹{2- _wandbruntime‚2553  _runtime‚2553‹7 ^Ò'‘ÌO TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63894547 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5467185310720 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚104108905878528 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5310100960888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63894547 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63884891 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚481 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40194729 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40194727 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚109186570577408 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚104108880974848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5310186764012 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚2553.553628878 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚109186675046400 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5323216167936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7858128083604681e+09 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23821245 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5467311790592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚103863458878464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104089276 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚109186675046400 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40194729 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17323930 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚109576066285568 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚106592791016844 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚106592688174360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41145427 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41145175 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚101282604252832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚109576217669120 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17324180 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63894547 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63884891 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚481 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40194729 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40194727 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚109186570577408 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚104108880974848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5310186764012 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚2553.553628878 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚109186675046400 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5323216167936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7858128083604681e+09 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23821245 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5467311790592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚252 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚103863458878464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104089276 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚109186675046400 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40194729 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17323930 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚109576066285568 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚106592791016844 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚106592688174360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41145427 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41145175 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚101282604252832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚109576217669120 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17324180 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41152004 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23825182 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5468219435008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41152275 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚103880608602112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5324095794688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40201372 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5324008357888 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63905084 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚109204721436160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚104126112995328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚271 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5324008357888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚106610408662498 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5311064237346 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103880608602112 trainer/global_step‚6049 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40201370 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 % _timestamp‚1.7858128087184072e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚109594332430336 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40201372 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚106610305819902 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40201370 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17327091 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104106456 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5324095794688 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104126088091648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 _runtime‚2553.908679749 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚103880625641472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5310978434110 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63895414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104106456 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚269 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41152004 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23825182 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5468219435008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_±½¾þÒ$train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41152275 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚103880608602112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5324095794688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40201372 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5324008357888 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63905084 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚109204721436160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚104126112995328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚271 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5324008357888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚106610408662498 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5311064237346 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103880608602112 trainer/global_step‚6049 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40201370 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 % _timestamp‚1.7858128087184072e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚109594332430336 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40201372 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚106610305819902 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40201370 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17327091 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104106456 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5324095794688 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104126088091648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 _runtime‚2553.908679749 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚103880625641472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5310978434110 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63895414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104106456 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚269 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104949428 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5512488745472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17467897 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚110088904863232 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5367212303360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5367212303360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚110481750464000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5367124873728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚102119612868832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40526877 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚107473688096352 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64422549 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5367124873728 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24018095 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64422549 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41485994 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚110088904863232 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5367212303360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5367212303360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚110481750464000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5367124873728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚102119612868832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40526877 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚107473688096352 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64422549 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5367124873728 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24018095 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64422549 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41485994 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104949428 MGDeviceStatsñš®Þ Monitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5512488745472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17467897 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5354866897618 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40533522 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64433086 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41492555 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚110499872008192 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚107491305742006 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚102136353041152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5354952700854 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104966608 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40533520 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5513268311552 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚104738946752512 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚104738946752512 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17470523 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚102136336001792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104986443931648 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚104738963791872 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚110107055721984 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 $ _timestamp‚1.785812829261955e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5513403172864 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚107491202899410 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5368091930112 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5368004493312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64423416 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104966608 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104956936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41492808 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚6099 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24022034 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104956936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚110499712243200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _step‚486 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24022032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5368091930112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _runtime‚ 2574.45222341ùoЈß&£ÙM $ _timestamp‚1.785812829261955e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5513403172864 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚107491202899410 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5368091930112 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5368004493312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64423416 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104966608 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104956936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41492808 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚6099 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24022034 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104956936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚110499712243200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _step‚486 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24022032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5368091930112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _runtime‚ 2574.45222341 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5368004493312 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64433086 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5354866897618 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40533522 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64433086 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41492555 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚110499872008192 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚107491305742006 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚102136353041152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5354952700854 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104966608 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40533520 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5513268311552 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚104738946752512 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚104738946752512 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17470523 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚102136336001792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104986443931648 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚104738963791872 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚110107055721984 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392æt»ð2¤- _wandbruntime‚2574  _runtime‚2574ÑR¥ÿ 0loss/FRt1/assignment_loss‚3.6462757440462505 2loss/SRqqt1/assignment_loss‚1.1607459120123702 1loss/FBt2/assignment_loss‚0.14878687267189564 1loss/SRqqt2/detection_loss‚2.0267117618493837 0loss/FRt2/assignment_loss‚3.5049542906609448 /loss/FRt1/detection_loss‚2.4614154151449466 /loss/FRt2/detection_loss‚2.5019725263118744 1loss/SRqqt1/detection_loss‚0.8837873742792427 /loss/FBt1/detection_loss‚2.2698440551757812 _runtime‚2574.452959889 _step‚487 2loss/SRqqt2/assignment_loss‚1.2449555188825685 0loss/FBt1/assignment_loss‚0.0772882213016904 .loss/FBt2/detection_loss‚1.373429837088371 epoch‚0 trainer/global_step‚6099 %loss/total_loss‚21.30016752942532 % _timestamp‚1.7858128292631547e+09ø) ¦„ 0loss/FRt1/assignment_loss‚3.6462757440462505 2loss/SRqqt1/assignment_loss‚1.1607459120123702 1loss/FBt2/assignment_loss‚0.14878687267189564 1loss/SRqqt2/detection_loss‚2.0267117618493837 0loss/FRt2/assignment_loss‚3.5049542906609448 /loss/FRt1/detection_loss‚2.4614154151449466 /loss/FRt2/detection_loss‚2.5019725263118744 1loss/SRqqt1/detection_loss‚0.8837873742792427 /loss/FBt1/detection_loss‚2.2698440551757812 _runtime‚2574.452959889 _step‚487 2loss/SRqqt2/assignment_loss‚1.2449555188825685 0loss/FBt1/assignment_loss‚0.0772882213016904 .loss/FBt2/detection_loss‚1.373429837088371 epoch‚0 trainer/global_step‚6099 %loss/total_loss‚21.30016752942532 % _timestamp‚1.7858128292631547e+09çX2Üœ§‚(:‘ â®ÅÓ€ˆ­ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚132.671' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.22366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ô 1n¨‚(:ã  â®Åӿ؃ network.recv‚402126 disk.drbd1828.out‚0% memory_percent‚41.235268115997314 disk.drbd1584.out‚0 disk.drbd1285.in‚0 disk.sda1.in‚0$ disk./.usageGB‚586.3553581237793 disk.sda1.out‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.rbd1.out‚ 10.4912109375 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0$ proc.memory.rssMB‚11254.05859375 proc.cpu.threads‚29 network.sent‚ 5.655832e+06 disk.drbd1356.in‚0 disk.drbd1965.out‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0) proc.memory.availableMB‚ 38512.0546875 disk.drbd1828.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1931.in‚0 disk.drbd1938.out‚0 disk.drbd1584.in‚0 disk.drbd1289.in‚0 disk.rbd1.in‚0 cpu‚13.786277532847478 disk.drbd1931.out‚0 disk.drbd2118.in‚0 disk.sda2.out‚ 84.23046875 disk.sda.in‚ 0.68359375 disk.sda.out‚ 84.23046875 disk.drbd1965.in‚0! disk.dm-0.out‚622.85205078125 disk.drbd1673.out‚0 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.rbd0.out‚0 disk.dm-0.in‚ 4.9296875 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1550.in‚0) disk./.usagePercent‚32.79206116815746 disk.drbd1552.in‚0 disk.nvme0n1.in‚ 4.9296875 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.drbd1226.in‚0$ disk.nvme0n1.out‚622.85205078125 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0Q¸Ž©‚(:„ ñ®ÅÓ£Ìþ‚ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚43.0 gpu.0.powerWatts‚131.94' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.98 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0=žfQmª‚(:â  ñ®ÅÓъă disk.drbd1673.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1285.out‚0 disk.drbd1550.in‚0 disk.drbd2111.in‚0 disk.sda2.out‚ 84.67578125 disk.drbd1356.in‚0 disk.sda.out‚ 84.67578125$ disk./.usageGB‚586.3553581237793 disk.drbd1828.in‚0 disk.drbd1289.in‚0 disk.drbd1673.in‚0 disk.rbd1.out‚ 10.4912109375$ proc.memory.rssMB‚11254.05859375 proc.cpu.threads‚29 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0! disk.dm-0.out‚622.98095703125 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 network.recv‚404568 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.drbd1828.out‚0 disk.drbd1552.out‚0 disk.drbd2078.in‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.drbd1226.out‚0 disk.drbd1817.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1510.out‚0 disk.drbd2118.in‚0 disk.nvme0n1.in‚ 4.9296875 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1552.in‚0 disk.drbd1938.out‚0 disk.drbd1931.in‚0$ memory_percent‚41.23564958572388$ disk.nvme0n1.out‚622.98095703125) proc.memory.availableMB‚ 38511.8046875 cpu‚13.785557241017546 network.sent‚ 5.700984e+06) disk./.usagePercent‚32.79206116815746 disk.drbd1965.out‚0 disk.dm-0.in‚ 4.9296875=º2«- _wandbruntime‚2595  _runtime‚2595ÀĹǬÁ & lr-AdamW/pg1‚0.0001584731746981472 & lr-AdamW/pg2‚0.0001584731746981472 trainer/global_step‚6149 % _timestamp‚1.7858128499972365e+09 _runtime‚2595.187163433 _step‚488q~e™Ì­Æ _step‚488 & lr-AdamW/pg1‚0.0001584731746981472 & lr-AdamW/pg2‚0.0001584731746981472 trainer/global_step‚6149 % _timestamp‚1.7858128499972365e+09 _runtime‚2595.187163433èã¤Ë2®- _wandbruntime‚2595  _runtime‚2595€tÙ^¯ËO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚110991239149056 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105809580 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚102956604445472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚ 2595.19179543 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5411121009152 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5411208438784 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40859029 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚105580135179264 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858128499981234e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚105580118139904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24214945 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚105580135179264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚253 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24214947 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40859027 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5557546551808 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5397877887904 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64940895 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMâʇFsonitor.on_train_batch_start/active.large_pool.allocated‚40859029 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚105580118139904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚110991343618048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚105829617558528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚108354482333376 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64940895 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚102956621484832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5557683517440 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40859027 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41826648 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41826395 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5397963691028 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚108354585175860 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 105799922 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64950551 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5411121009152 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚105829592654848 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5411208438784 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392ÏVæÖ'°ÐO PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40859027 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5557546551808 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5397877887904 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64940895 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40859029 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚105580118139904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚110991343618048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚105829617558528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚108354482333376 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64940895 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚102956621484832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5557683517440 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40859027 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41826648 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41826395 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5397963691028 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚108354585175860 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 105799922 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64950551 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5411121009152 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚105829592654848 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5411208438784 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚110991239149056 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105809580 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚102956604445472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚ 2595.19179543 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5411121009152 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5411208438784 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40859029 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚105580135179264 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858128499981234e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚105580118139904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24214945 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚105580135179264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚253 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24214947é{Ô#è2±- _wandbruntime‚2595  _runtime‚2595_ÌeXÛ&²ÕM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40865670 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚105597284902912 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5412088065536 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105826760 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 105817088 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚105846799771648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚102973344617792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5412088065536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚111405246340096 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚2595.533832155 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5398841164362 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5558446568448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64951418 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105826760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚102973361657152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5398755361126 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40865670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41833196 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64961088 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105817088 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚105846824675328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚6149 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40865672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5412000628736 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚490 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚105597284902912 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5412000628736 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111009285531648 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚108372202821514 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251ê­W‡³ÚM H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 105817088 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚105846799771648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚102973344617792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5412088065536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚111405246340096 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚2595.533832155 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5398841164362 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5558446568448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64951418 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105826760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚102973361657152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5398755361126 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40865670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41833196 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64961088 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105817088 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚105846824675328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚6149 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40865672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5412000628736 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚490 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚105597284902912 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5412000628736 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111009285531648 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚108372202821514 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40865670 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚105597284902912 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5412088065536 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H:DeviceStatsMonitor.on_train…µìJY _batch_end/active.all.allocated‚ 105826760 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936êçMˆ72´- _wandbruntime‚2595  _runtime‚2595LOgµ€ 0loss/FRt1/assignment_loss‚3.4269939019129825 /loss/FBt2/detection_loss‚1.0733023158616721 &loss/total_loss‚20.646585214215833 _runtime‚2595.534514019 1loss/FBt1/assignment_loss‚0.18410473338262817 1loss/SRqqt1/detection_loss‚1.0405618873305786 epoch‚0 $ _timestamp‚1.785812850344716e+09 _step‚491 2loss/SRqqt1/assignment_loss‚1.3563810873048423 2loss/SRqqt2/assignment_loss‚1.3027575255837291 1loss/FBt2/assignment_loss‚0.16592007781847018 /loss/FRt1/detection_loss‚2.2895395428269776 .loss/FRt2/detection_loss‚2.541442958535729 1loss/SRqqt2/detection_loss‚2.2695334531599656 trainer/global_step‚6149 0loss/FRt2/assignment_loss‚3.6269516420888377 /loss/FBt1/detection_loss‚1.3690960884094239³¤Ò ¶… _runtime‚2595.534514019 1loss/FBt1/assignment_loss‚0.18410473338262817 1loss/SRqqt1/detection_loss‚1.0405618873305786 epoch‚0 $ _timestamp‚1.785812850344716e+09 _step‚491 2loss/SRqqt1/assignment_loss‚1.3563810873048423 2loss/SRqqt2/assignment_loss‚1.3027575255837291 1loss/FBt2/assignment_loss‚0.16592007781847018 /loss/FRt1/detection_loss‚2.2895395428269776 .loss/FRt2/detection_loss‚2.541442958535729 1loss/SRqqt2/detection_loss‚2.2695334531599656 trainer/global_step‚6149 0loss/FRt2/assignment_loss‚3.6269516420888377 /loss/FBt1/detection_loss‚1.3690960884094239 0loss/FRt1/assignment_loss‚3.4269939019129825 /loss/FBt2/detection_loss‚1.0733023158616721 &loss/total_loss‚20.646585214215833ëùÑš·‚(: €¯ÅÓøÖ¡ƒ gpu.0.gpu‚6.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚97.99' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚32.663333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0È[díe¸‚(:Ú  €¯ÅÓÜú¼ƒ disk.drbd1510.in‚0 disk.rbd0.in‚0 disk.drbd2111.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1550.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd2078.in‚0 disk.rbd1.out‚ 10.5234375 disk.drbd1965.in‚0 disk.drbd2118.out‚0 disk.sda2.in‚ 0.68359375# disk.nvme0n1.out‚623.4697265625 disk.drbd1828.out‚0 disk.sda2.out‚ 84.9765625 disk.dm-0.in‚ 4.9296875 disk.drbd1289.in‚0 disk.drbd1356.out‚0 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.drbd1226.in‚0$ memory_percent‚41.23536944389343 disk.drbd2118.in‚0 disk.drbd2078.out‚0* proc.memory.availableMB‚38511.98828125 proc.cpu.threads‚29 cpu‚13.78470363670784$ disk./.usageGB‚586.3553771972656 disk.drbd1673.in‚0 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.rbd0.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1828.in‚0 disk.drbd1510.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 network.sent‚ 5.70405e+06 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.drbd1938.in‚0 network.recv‚405584 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.drbd1584.in‚0 disk.drbd1550.in‚0) disk./.usagePercent‚32.79206223484666 disk.rbd1.in‚0 disk.drbd1938.out‚0 disk.dm-0.out‚623.4697265625 disk.drbd1285.out‚0 disk.drbd1552.in‚0 disk.nvme0n1.in‚ 4.9296875 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd1285.in‚0 disk.sda.out‚ 84.9765625r5q©2¹- _wandbruntime‚2616  _runtime‚2616ãǺÁ _runtime‚2616.020609136 _step‚492 & lr-AdamW/pg1‚0.0001597617840224125 & lr-AdamW/pg2‚0.0001597617840224125 trainer/global_step‚6199 % _timestamp‚1.7858128708292012e+09)*?èÌ»Æ & lr-AdamW/pg1‚0.0001597617840224125 & lr-AdamW/pg2‚0.0001597617840224125 trainer/global_step‚6199 % _timestamp‚1.7858128708292012e+09 _runtime‚2616.020609136 _step‚492ìî¬Úv2¼- _wandbruntime‚2616  _runtime‚2616j&K(Ò'½ÌO TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65478553 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65468897 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17754242 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚106438456290304 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚6199 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5602705220096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 % _timestamp‚1.7858128708298798e+09 _step‚493 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚106438473329664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106660074 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106669732 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42165774 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41191179 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚111893573434880 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚109235482255368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚265 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24411795 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚112292653714944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚109235379412884 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65478553 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚111893677903872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5455204574208 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚106438473329664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚106689948494848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚111893573434880 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5455117144576 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚263 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17753979 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5441852154536 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0p•”×'¾ÑO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚111893677903872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5455204574208 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚106438473329664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚106689948494848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚111893573434880 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5455117144576 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚263 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17753979 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5441852154536 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65478553 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65468897 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17754242 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚106438456290304 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚6199 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5602705220096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 % _timestamp‚1.7858128708298798e+09 _step‚493 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚106438473329664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106660074 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106669732 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42165774 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41191179 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚111893573434880 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚109235482255368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚265 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24411795 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚112292653714944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚109235379412884 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65478553 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0íšÛC<2¿- _wandbruntime‚2616  _runtime‚26164ÇéK1ÀÖM REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17756690 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5456084200960 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24415734 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106686912 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42172422 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5442729627870 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5442643824634 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚106707180515328 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5455996764160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚112310918974976 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65489090 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65489090 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65479420 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42172664 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚103810370273152 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41197820 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚109253099901022 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚103810353233792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMoni»>N«tor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65479420 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5456084200960 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚111911724293632 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚242 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5603603598336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚106455623053312 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24415732 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚112310759209984 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 % _timestamp‚1.7858128711812599e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111911619817472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106686912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17756930 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚106455640092672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚6199 _step‚494 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106677240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41197820 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚109252997058426 _runtime‚2616.371532206{¨á/á&ÁÛM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚111911724293632 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚242 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5603603598336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚106455623053312 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24415732 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚112310759209984 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 % _timestamp‚1.7858128711812599e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111911619817472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106686912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17756930 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚106455640092672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚6199 _step‚494 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106677240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41197820 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚109252997058426 _runtime‚2616.371532206 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17756690 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5456084200960 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24415734 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106686912 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42172422 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5442729627870 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5442643824634 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚106707180515328 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5455996764160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚112310918974976 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65489090 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65489090 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65479420 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42172664 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚103810370273152 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41197820 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚109253099901022 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚103810353233792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65479420 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5456084200960 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41523327 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚113198144151040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚107550329238528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107550304334848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66006555 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42505222 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5647839816192 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 107529884 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚113198306020352 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚107296811480064 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚112796012189696 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66006555 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚110116379334876 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚107296811480064 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5499200709632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5499113280000 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41523329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17896577 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24608645 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65996899 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42505467 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚6249 _step‚497 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 % _timestamp‚1.7858128918570037e+09 _runtime‚ 2637.04770808 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚107296794440704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24608647 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17896820 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110116276492392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚112795907720704 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5647976781824 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5485654814920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65996899 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5499113280000 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚104630621677472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚112795907720704 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5499113280000 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41523329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17896577 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24608645 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65996899 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42505467 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚6249 _step‚497 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 % _timestamp‚1.7858128918570037e+09 _runtime‚ 2637.04770808 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚107296794440704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24608647 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17896820 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110116276492392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚112795907720704 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5647976781824 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDevi¦»RÆceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5485654814920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65996899 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5499113280000 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚104630621677472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚112795907720704 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41523327 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚113198144151040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚107550329238528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107550304334848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66006555 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42505222 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5647839816192 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 107529884 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚113198306020352 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚107296811480064 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚112796012189696ñ#òÍ@2Í- _wandbruntime‚2637  _runtime‚26379ºë Ü&ÎÖM JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41529970 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚266 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚107313961203712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42512284 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5499992899584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24612584 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚6249 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5499992899584 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚113216251382784 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 107537392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚112814058579456 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚107313978243072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42512016 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚104647378889152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚107567511451648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚104647361849792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚2637.400336457 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚112813954103296 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5500080336384 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41529972 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 % _timestamp‚1.7858128922099793e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66007422 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107547064 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66017092 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66007422 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17899700 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚107567536355328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _step‚498 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚113216415342080 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41529972 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 107547064 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41529970 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚107313978243072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚107313961203712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5486618091378 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚110133894137934 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24612582 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640‚iá&ÏÛM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚107313961203712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42512284 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5499992899584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24612584 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚6249 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5499992899584 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚113216251382784 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 107537392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚112814058579456 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚107313978243072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42512016 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚104647378889152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚107567511451648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚104647361849792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚2637.400336457 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚112813954103296 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5500080336384 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41529972 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 % _timestamp‚1.7858128922099793e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66007422 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107547064 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66017092 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66007422 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17899700 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚107567536355328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _step‚498 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚113216415342080 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41529972 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 107547064 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41529970 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚107313978243072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚107313961203712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5486618091378 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚110133894137934 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24612582 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41529970 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚266 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0òÜÚ>¦2Ð- _wandbruntime‚2637  _runtime‚2637 A™gÑÿ 1loss/FBt2/assignment_loss‚0.04680395944843284 0loss/FRt2/assignment_loss‚3.4573393792522196 /loss/FRt2/detection_loss‚2.3670805607523238 .loss/FBt2/detection_loss‚1.277167589509521 2loss/SRqqt2/assignment_loss‚1.4651909839982789 1loss/FBt1/assignment_loss‚0.34759227890650174 .loss/FRt1/detection_loss‚2.423017514935311 1loss/SRqqt2/detection_loss‚2.3027065854209163 &loss/total_loss‚20.185879366920624 epoch‚0 trainer/global_step‚6249 % _timestamp‚1.7858128922113523e+09 2loss/SRqqt1/assignment_loss‚1.1287468453007403 0loss/SRqqt1/detection_loss‚0.887538863895044 /loss/FBt1/detection_loss‚1.0820891563946913 _runtime‚2637.401187927 _step‚499 0loss/FRt1/assignment_loss‚3.4006056491066428 Âph Ò„ _step‚499 0loss/FRt1/assignment_loss‚3.4006056491066428 1loss/FBt2/assignment_loss‚0.04680395944843284 0loss/FRt2/assignment_loss‚3.4573393792522196 /loss/FRt2/detection_loss‚2.3670805607523238 .loss/FBt2/detection_loss‚1.277167589509521 2loss/SRqqt2/assignment_loss‚1.4651909839982789 1loss/FBt1/assignment_loss‚0.34759227890650174 .loss/FRt1/detection_loss‚2.423017514935311 1loss/SRqqt2/detection_loss‚2.3027065854209163 &loss/total_loss‚20.185879366920624 epoch‚0 trainer/global_step‚6249 % _timestamp‚1.7858128922113523e+09 2loss/SRqqt1/assignment_loss‚1.1287468453007403 0loss/SRqqt1/detection_loss‚0.887538863895044 /loss/FBt1/detection_loss‚1.0820891563946913 _runtime‚2637.401187927óÝN‘Ó‚(:† ž¯ÅÓ£àø‚ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚122.313' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚40.771 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ž`fÔ‚(:Û  ž¯ÅÓöÝÀƒ disk.drbd1938.out‚0 disk.sda2.in‚ 0.68359375* proc.memory.percent‚17.172330617904663 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.dm-0.out‚623.8603515625 disk.rbd1.in‚0 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.drbd1285.in‚0$ proc.memory.rssMB‚11254.05859375 proc.cpu.threads‚29 disk.rbd0.in‚0 disk.drbd1552.out‚0 disk.drbd1356.out‚0$ disk./.usageGB‚586.3554000854492 disk.drbd2111.out‚0 disk.drbd1510.out‚0 disk.nvme0n1.in‚ 4.9296875 disk.drbd2118.out‚0 disk.drbd1584.out‚0 disk.drbd1226.in‚0 disk.sda2.out‚ 85.66015625 disk.rbd1.out‚ 10.6796875 disk.drbd1584.in‚0 disk.drbd2041.out‚0 disk.drbd1510.in‚0 disk.sda1.out‚0 network.recv‚410484) disk./.usagePercent‚32.79206351487371 disk.drbd1828.in‚0 disk.drbd1817.out‚0$ memory_percent‚41.23682975769043 disk.drbd1289.in‚0# disk.nvme0n1.out‚623.8603515625 disk.drbd1931.out‚0 disk.sda.in‚ 0.68359375 disk.rbd0.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.drbd2078.out‚0 disk.dm-0.in‚ 4.9296875 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.drbd1673.out‚0 network.sent‚ 5.794173e+06 disk.drbd1552.in‚0' proc.memory.availableMB‚ 38511.03125 disk.drbd1550.out‚0 disk.drbd1938.in‚0 disk.drbd1356.in‚0 cpu‚13.782792135108796 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd1285.out‚0 disk.sda1.in‚0 disk.sda.out‚ 85.66015625 disk.drbd2111.in‚0OÛ-åœÕ‚(:‘ ­¯ÅÓ¸Ø΃ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚42.0 gpu.0.powerWatts‚131.384' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.79466666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0˜ß#‡jÖ‚(:ß  ­¯ÅÓß×რdisk.drbd2041.out‚0 disk.sda.out‚ 85.94921875 disk.sda2.out‚ 85.94921875 disk.nvme0n1.in‚ 4.9296875 disk.drbd1965.out‚0 disk.drbd2111.out‚0 disk.drbd1673.in‚0 disk.drbd1285.in‚0$ memory_percent‚41.23660922050476( disk./.usagePercent‚32.7920613814953 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.rbd0.in‚0 proc.cpu.threads‚29! disk.dm-0.out‚624.14404296875 disk.sda1.out‚0 disk.drbd2118.out‚0 network.recv‚412944 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.dm-0.in‚ 4.9296875$ disk./.usageGB‚586.3553619384766 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.drbd1938.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1226.in‚0 disk.drbd2078.in‚0 disk.drbd1817.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 network.sent‚ 5.839206e+06 disk.sda1.in‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11254.05859375 disk.rbd1.out‚ 10.6796875 disk.drbd1931.in‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0* proc.memory.percent‚17.172330617904663 disk.rbd1.in‚0 disk.drbd1550.out‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1552.out‚0* proc.memory.availableMB‚38511.17578125$ disk.nvme0n1.out‚624.14404296875 disk.drbd1356.out‚0 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd1510.out‚0 cpu‚13.782148523554548 disk.drbd1356.in‚0×Ôð2×- _wandbruntime‚2658  _runtime‚2658ù¸-ÔÇØÁ & lr-AdamW/pg1‚0.0001623390026709431 & lr-AdamW/pg2‚0.0001623390026709431 trainer/global_step‚6299 % _timestamp‚1.7858129130750208e+09 _runtime‚2658.264964764 _step‚500fDÃÌÙÆ & lr-AdamW/pg1‚0.0001623390026709431 & lr-AdamW/pg2‚0.0001623390026709431 trainer/global_step‚6299 % _timestamp‚1.7858129130750208e+09 _runtime‚2658.264964764 _step‚500ôU _Ž2Ú- _wandbruntime‚2658  _runtime‚26586¤¦ÛÌO JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24805497 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5543196845056 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18040506 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41855477 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚113698346475520 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚110997276414384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858129130759706e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24805495 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66534557 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5692989968896 _runtime‚2658.269626849 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚113698242006528 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42846001 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚108410660174848 trainer/global_step‚6299 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5543196845056 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5529543278428 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚108155132591104 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66534557 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚105467647332832 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41855479 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5693120643072 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18040758 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110997173571900 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108380378 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41855477 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5543109415424 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5543109415424 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41855479 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚114103650143744 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚105467630293472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66524901 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚113698242006528 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚113698346475520 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚110997276414384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858129130759706e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24805495 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66534557 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5692989968896 _runtime‚2658.269626849 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚113698242006528 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42846001 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚108410660174848 trainer/global_step‚6299 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5543196845056 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5529543278428 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚108155132591104 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66534557 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚105467647332832 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41855479 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5693120643072 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18040758 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110997173571900 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108380378 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41855477 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5543109415424 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5543109415424 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41855479 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚114103650143744 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚105467630293472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66524901 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚113698242006528 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24805497 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5543196845056 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18040506 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251õ®á5”2Ý- _wandbruntime‚2658  _runtime‚2658¼¢/Û&ÞÕM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5543989035008 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚108427892195328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5693893377024 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24809434 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5694026141184 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108407216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5530506554886 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24809432 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 108407216 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚108172299354112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚269 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚111014791217442 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41862122 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚114121760668672 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 2658.61598949 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 % _timestamp‚1.7858129134257157e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66535424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5544076471808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18043295 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚113716392865280 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66535424 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚113716288389120 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5543989035008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _step‚502 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚108172316393472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5544076471808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚108427867291648 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42852727 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66545094 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚108172316393472 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚108172299354112 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108407216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5530506554886 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24809432 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 108407216 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚108172299354112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚269 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚111014791217442 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41862122 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚114121760668672 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 2658.61598949 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 % _timestamp‚1.7858129134257157e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66535424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5544076471808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18043295 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/all˜úocated_bytes.all.allocated‚113716392865280 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66535424 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚113716288389120 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5543989035008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _step‚502 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚108172316393472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5544076471808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚108427867291648 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42852727 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66545094 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚108172316393472 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚108172299354112 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24809434 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5694026141184 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚114600576292352 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5587192980480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5587105550848 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚238 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚115009218602496 _step‚505 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 109240530 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18182242 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚114600680761344 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚109271040918528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚6349 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42187627 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5587105550848 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67062559 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚115009369986048 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25002345 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25002347 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5587192980480 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43184351 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚114600576292352 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5587192980480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5587105550848 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚238 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚115009218602496 _step‚505 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 109240530 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18182242 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚114600680761344 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚109271040918528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚6349 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42187627 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5587105550848 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67062559 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚115009369986048 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25002345 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25002347 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5587192980480 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43184351 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109267368 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch©:ÏP#_end/inactive_split_bytes.large_pool.peak‚86278144 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43191372 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25006282 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67073096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5588072607232 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5587985170432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67063426 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚115027485976064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚67063426 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5587985170432 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18185088 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚106321379081792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5739237940736 trainer/global_step‚6349 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚109030654543872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42194272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42194270 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚109288248035328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109257696 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚114618727151104 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚115027328308224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43191119 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5574395018394 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚506 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42194270 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5588072607232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚111895791139546 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚109030654543872 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚109030637504512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5574309215158 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42194272 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25006284 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚109288223131648 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚106321396121152 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚114618727151104 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5739105176576 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109267368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18184837 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121BP‰†à&íÚM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42194272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42194270 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚109288248035328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109257696 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚114618727151104 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚115027328308224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43191119 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5574395018394 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚506 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42194270 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5588072607232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚111895791139546 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚109030654543872 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚109030637504512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5574309215158 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42194272 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25006284 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚109288223131648 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚106321396121152 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚114618727151104 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5739105176576 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109267368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18184837 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109267368 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43191372 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25006282 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67073096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5588072607232 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5587985170432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67063426 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚115027485976064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚67063426 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5587985170432 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18185088 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚106321379081792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5739237940736 trainer/global_step‚6349 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚109030654543872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0ú3œh2î- _wandbruntime‚2679  _runtime‚2679þB—õïü 1loss/FBt1/assignment_loss‚0.34668030199827626 0loss/SRqqt1/detection_loss‚1.065966895538139 /loss/FRt1/assignment_loss‚3.562377388988222 1loss/FBt2/assignment_loss‚0.04615856497722031 .loss/FRt1/detection_loss‚2.475689048923197 /loss/FBt2/detection_loss‚1.2859470566117024 %loss/total_loss‚22.85458288619483 % _timestamp‚1.7858129341865518e+09 0loss/FRt2/assignment_loss‚3.3266682217392742 2loss/SRqqt1/assignment_loss‚1.2443805050675396 -loss/FRt2/detection_loss‚2.62675547147099 _runtime‚2679.376340303 epoch‚0 2loss/SRqqt2/assignment_loss‚1.5086303814672506 1loss/SRqqt2/detection_loss‚2.0644258483098104 /loss/FBt1/detection_loss‚3.3009032011032104 trainer/global_step‚6349 _step‚5072Œ™Nð /loss/FRt1/assignment_loss‚3.562377388988222 1loss/FBt2/assignment_loss‚0.04615856497722031 .loss/FRt1/detection_loss‚2.475689048923197 /loss/FBt2/detection_loss‚1.2859470566117024 %loss/total_loss‚22.85458288619483 % _timestamp‚1.7858129341865518e+09 0loss/FRt2/assignment_loss‚3.3266682217392742 2loss/SRqqt1/assignment_loss‚1.2443805050675396 -loss/FRt2/detection_loss‚2.62675547147099 _runtime‚2679.376340303 epoch‚0 2loss/SRqqt2/assignment_loss‚1.5086303814672506 1loss/SRqqt2/detection_loss‚2.0644258483098104 /loss/FBt1/detection_loss‚3.3009032011032104 trainer/global_step‚6349 _step‚507 1loss/FBt1/assignment_loss‚0.34668030199827626 0loss/SRqqt1/detection_loss‚1.065966895538139û?襭‘ñ‚(:† ˯ÅÓÕø‚ gpu.0.gpu‚31.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚122.349' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚40.783 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“‚Hžgò‚(:Ü  ˯ÅÓÕí¼ƒ disk.drbd1584.in‚0 disk.rbd0.in‚0 disk.drbd1938.in‚0 disk.drbd1828.in‚0 disk.dm-0.out‚624.6240234375 disk.drbd2041.out‚0 disk.drbd2111.in‚0 disk.sda1.in‚0 disk.drbd2041.in‚0 disk.drbd2111.out‚0 disk.drbd1289.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1817.in‚0 disk.rbd1.out‚ 10.8056640625 disk.drbd1931.out‚0 disk.drbd1226.in‚0 disk.drbd1285.in‚0 disk.drbd1510.in‚0 disk.drbd2118.out‚0$ memory_percent‚41.23713970184326 disk.drbd2078.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.drbd1356.out‚0( proc.memory.availableMB‚ 38510.828125 disk.drbd1938.out‚0 cpu‚13.780845359957093( disk./.usagePercent‚32.7920613814953 disk.sda.out‚ 86.6640625 disk.drbd1226.out‚0 disk.rbd0.out‚0 network.recv‚416128 disk.drbd1550.in‚0 disk.drbd1510.out‚0 disk.sda2.in‚ 0.68359375 disk.sda2.out‚ 86.6640625$ disk./.usageGB‚586.3553619384766 disk.nvme0n1.in‚ 4.9296875 network.sent‚ 5.887313e+06 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1289.in‚0 disk.drbd1817.out‚0 disk.drbd1356.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.dm-0.in‚ 4.9296875 proc.cpu.threads‚29 disk.drbd1828.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0# disk.nvme0n1.out‚624.6240234375 disk.drbd1550.out‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0$ proc.memory.rssMB‚11254.05859375ò•0Ã2ó- _wandbruntime‚2699  _runtime‚26992íÉÇôÁ % _timestamp‚1.7858129545357475e+09 _runtime‚2699.725688271 _step‚508 & lr-AdamW/pg1‚0.0001649162213194737 & lr-AdamW/pg2‚0.0001649162213194737 trainer/global_step‚6399XŽ2ãÌõÆ & lr-AdamW/pg2‚0.0001649162213194737 trainer/global_step‚6399 % _timestamp‚1.7858129545357475e+09 _runtime‚2699.725688271 _step‚508 & lr-AdamW/pg1‚0.0001649162213194737ün ›2ö- _wandbruntime‚2699  _runtime‚2699𠽊R%÷ÌO eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚110131396758528 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚115502910578176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚115502910578176 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚67580905 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5617320205444 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚115503015047168 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚115914725084160 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚6399 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _runtime‚2699.730637597 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5631101686272 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5631101686272 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚112758967730916 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5783353229312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚107141664564832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚107141647525472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67590561 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚109871808891904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5631189115904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110131371854848 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18324774 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚112759070573400 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚115503015047168 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43523704 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42519777 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43523971 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚109871825931264C çˆ×'øÑO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚115503015047168 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚115914725084160 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚6399 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _runtime‚2699.730637597 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5631101686272 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5631101686272 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚112758967730916 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5783353229312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚107141664564832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚107141647525472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67590561 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚109871808891904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5631189115904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110131371854848 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18324774 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚112759070573400 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚115503015047168 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43523704 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42519777 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43523971 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚109871825931264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚110131396758528 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚115502910578176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚115502910578176 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚67580905 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5617320205444ýÒÉ»)‘ù‚(:† Ú¯ÅÓÊù‚ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚103.848' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚34.616 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0wpbú‚(:×  Ú¯ÅÓ×”¾ƒ disk.drbd1965.in‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0 disk.drbd1584.in‚0 disk.sda2.out‚87.3125( disk./.usagePercent‚32.7920613814953 disk.sda.in‚ 0.68359375 disk.drbd1285.out‚0 disk.sda1.out‚0* proc.memory.availableMB‚38510.57421875 disk.drbd1931.in‚0 disk.drbd1510.in‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd1828.out‚0 cpu‚13.780089711079897 proc.cpu.threads‚29 disk.drbd1356.in‚0 disk.drbd1965.out‚0 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd1931.out‚0 network.recv‚418646 disk.drbd1226.out‚0 disk.drbd1552.out‚0# memory_percent‚41.2375271320343 disk.drbd2041.out‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2118.out‚0# disk.nvme0n1.out‚624.8388671875 disk.drbd1828.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1584.out‚0$ disk./.usageGB‚586.3553619384766 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.sda.out‚87.3125 disk.dm-0.in‚ 4.9296875 disk.drbd2118.in‚0 disk.nvme0n1.in‚ 4.9296875 disk.drbd1550.out‚0 disk.dm-0.out‚624.8388671875 disk.sda1.in‚0 disk.drbd1673.in‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd2111.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 10.8681640625* proc.memory.percent‚17.172330617904663 disk.drbd1510.out‚0 network.sent‚ 5.932376e+06ê’·ž2û- _wandbruntime‚2700  _runtime‚2700ŦǗÛ&üÕM F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18327341 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚109888975654912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5632068742656 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18327595 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67601098 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67591428 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚6399 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42526420 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42526422 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42526422 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚115932992834560 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5784388959232 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110127520 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚112776585376458 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110117848 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110117848 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚110148603875328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 110127520 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43530473 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _runtime‚2700.072641255 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚115520956960768 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚109888992694272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5631981305856 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚115521061436928 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5618197678666 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚67601098 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43530729 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚112776688219054 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚115521061436928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5632068742656 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5631981305856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42526420 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785812954882386e+09 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚110148578971648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚115932833069568 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚510;NÆdx&ýÚM D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚6399 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42526420 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42526422 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42526422 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚115932992834560 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5784388959232 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110127520 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚112776585376458 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110117848 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110117848 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚110148603875328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 110127520 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43530473 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _runtime‚2700.072641255 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚115520956960768 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚109888992694272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5631981305856 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚115521061436928 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5618197678666 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚67601098 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43530729 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚112776688219054 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚115521061436928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5632068742656 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5631981305856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42526420 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.785812954882386e+09 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚110148578971648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚115932833069568 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚510 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18327341 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚109888975654912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5632068742656 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18327595 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67601098 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67591428 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_splitdJh.large_pool.peak‚18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538þv A2þ- _wandbruntime‚2700  _runtime‚2700"> ÿþ 2loss/SRqqt1/assignment_loss‚1.2224612823885046 1loss/SRqqt1/detection_loss‚1.0304208057509228 _runtime‚2700.073325235 /loss/FRt2/assignment_loss‚3.390168623868809 1loss/FBt1/assignment_loss‚0.08076599029504905 2loss/FBt2/assignment_loss‚0.007780731105897881 /loss/FBt2/detection_loss‚1.3110956368839433 epoch‚0 _step‚511 /loss/FRt1/detection_loss‚2.4947180187231615 /loss/FRt2/detection_loss‚2.4858933278294497 /loss/FBt1/detection_loss‚1.6322891414165497 %loss/total_loss‚20.48349499704534 # _timestamp‚1.78581295488357e+09 2loss/SRqqt2/assignment_loss‚1.4149710444673416 1loss/SRqqt2/detection_loss‚1.8137664589430056 trainer/global_step‚6399 /loss/FRt1/assignment_loss‚3.5991639353727044"( €ƒ /loss/FRt1/assignment_loss‚3.599163935372704 2loss/SRqqt1/assignment_loss‚1.2224612823885046 1loss/SRqqt1/detection_loss‚1.0304208057509228 _runtime‚2700.073325235 /loss/FRt2/assignment_loss‚3.390168623868809 1loss/FBt1/assignment_loss‚0.08076599029504905 2loss/FBt2/assignment_loss‚0.007780731105897881 /loss/FBt2/detection_loss‚1.3110956368839433 epoch‚0 _step‚511 /loss/FRt1/detection_loss‚2.4947180187231615 /loss/FRt2/detection_loss‚2.4858933278294497 /loss/FBt1/detection_loss‚1.6322891414165497 %loss/total_loss‚20.48349499704534 # _timestamp‚1.78581295488357e+09 2loss/SRqqt2/assignment_loss‚1.4149710444673416 1loss/SRqqt2/detection_loss‚1.8137664589430056 trainer/global_step‚6399ÿíÛO˜œ‚(:‘ é¯ÅÓÉÖä‚ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚130.214' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.404666666666664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Pö]!h‚‚(:Ý  é¯ÅÓñˆ¬ƒ disk.drbd1938.out‚0 disk.rbd1.out‚ 10.9541015625 disk.drbd1584.in‚0 disk.drbd1226.out‚0 disk.drbd1285.in‚0 disk.sda.in‚ 0.68359375 proc.cpu.threads‚29 disk.sda1.out‚0 disk.sda2.in‚ 0.68359375$ disk.nvme0n1.out‚625.28955078125 disk.sda.out‚ 87.609375 disk.drbd2118.out‚0 disk.drbd1828.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1931.out‚0) proc.memory.availableMB‚ 38510.6484375( disk./.usagePercent‚32.7920613814953 disk.drbd1552.out‚0 disk.drbd1673.in‚0$ disk./.usageGB‚586.3553619384766 disk.rbd0.out‚0 disk.drbd1289.out‚0 disk.rbd1.in‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd1226.in‚0 disk.drbd1965.out‚0 network.recv‚421111 disk.drbd1550.in‚0 disk.sda1.in‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd1931.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1550.out‚0$ memory_percent‚41.23741388320923 cpu‚13.779478750619273 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd1356.in‚0 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.dm-0.in‚ 4.9296875 disk.drbd1510.out‚0 disk.nvme0n1.in‚ 4.9296875 disk.drbd1965.in‚0! disk.dm-0.out‚625.28955078125 disk.sda2.out‚ 87.609375 disk.drbd1356.out‚0 disk.drbd2041.in‚0 network.sent‚ 5.977602e+06 disk.drbd1289.in‚0 disk.drbd2118.in‚0&iΩ2ƒ- _wandbruntime‚2720  _runtime‚2720é20É„Ã trainer/global_step‚6449 % _timestamp‚1.7858129751248426e+09 _runtime‚2720.314785712 _step‚512 ' lr-AdamW/pg1‚0.00016620483064373902 ' lr-AdamW/pg2‚0.00016620483064373902|`ãÎ…È _runtime‚2720.314785712 _step‚512 ' lr-AdamW/pg1‚0.00016620483064373902 ' lr-AdamW/pg2‚0.00016620483064373902 trainer/global_step‚6449 % _timestamp‚1.7858129751248426e+09€ºðev2†- _wandbruntime‚2720  _runtime‚2720À†»êÒ'‡ÌO NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚68118563 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 110970492 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 trainer/global_step‚6449 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 110960834 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42851927 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5675097821696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43864077 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5661208668952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚68108907 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚110991752598528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5675097821696 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚110730147042304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42851929 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116405349332992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚116820398282752 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚113639967652908 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚113639864810424 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚116405349332992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43864309 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚68108907 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110991727694848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚107978673180832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 % _timestamp‚1.7858129751257677e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116405244864000 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5675185251328 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42851929 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42851927 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5675185251328 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25396047 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5828515010048 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5661294472076 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25396045 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18468262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18468032ìÅ\µ×'ˆÑO NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42851929 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116405349332992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚116820398282752 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚113639967652908 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚113639864810424 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚116405349332992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43864309 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚68108907 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110991727694848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚107978673180832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 % _timestamp‚1.7858129751257677e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116405244864000 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5675185251328 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42851929 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42851927 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5675185251328 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25396047 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5828515010048 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5661294472076 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25396045 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18468262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18468032 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚68118563 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 110970492 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 trainer/global_step‚6449 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 110960834 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42851927 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5675097821696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43864077 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5661208668952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚68108907 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚110991752598528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5675097821696 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚110730147042304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640_\èÍ2‰- _wandbruntime‚2720  _runtime‚27203à ŠÕM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42858570 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5829563593216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18470895 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚110747313805312 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚116838365640704 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚68119430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43871128 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚113657585298562 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 $ _timestamp‚1.785812975470161e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚110747313805312 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5676064878080 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5675977441280 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚113657482455966 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚116423291246592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5676064878080 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚68129100 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚110747330844672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18471144 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110987672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43870877 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚116838523308544 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚107995413353152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42858572 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚107995396313792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚116423395722752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚116423395722752 _step‚514 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110978000 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚6449 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25399984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚249 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110978000 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42858572 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XFDeviceStatsMonitor.on_train_batcßj„#¾h_end/requested_bytes.small_pool.freed‚ 5662086142174 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚68129100 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5829430829056 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚68119430 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 110987672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚2720.660419761 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚111008959715328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25399982 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42858570 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚2720.660419761 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚111008959715328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25399982 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42858570 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18470895 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚110747313805312 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚116838365640704 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚68119430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43871128 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚113657585298562 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 $ _timestamp‚1.785812975470161e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚110747313805312 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5676064878080 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5675977441280 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚113657482455966 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚116423291246592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5676064878080 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚68129100 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚110747330844672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18471144 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110987672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43870877 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚116838523308544 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚107995413353152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42858572 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚107995396313792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚116423395722752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚116423395722752 _step‚514 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110978000 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚6449 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25399984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚249 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110978000 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42858572 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5662086142174‚ÃÅC2Œ- _wandbruntime‚2720  _runtime‚2720šø!ÿ 1loss/SRqqt2/detection_loss‚1.9814287840587195 trainer/global_step‚6449 0loss/FRt1/assignment_loss‚3.4638357812708076 /loss/FRt1/detection_loss‚2.1628279713067142 .loss/FBt2/detection_loss‚1.253425591139655 &loss/total_loss‚20.883968209193647 _step‚515 0loss/FRt2/assignment_loss‚3.2392524416728685 1loss/FBt2/assignment_loss‚0.09460819296337471 1loss/SRqqt1/detection_loss‚1.0394227783588215 .loss/FBt1/detection_loss‚2.615449649592241 _runtime‚2720.661390009 2loss/SRqqt1/assignment_loss‚1.3820629994238705 epoch‚0 $ _timestamp‚1.785812975471623e+09 2loss/SRqqt2/assignment_loss‚1.2383751947066335 2loss/FBt1/assignment_loss‚0.005418609707703581 .loss/FRt2/detection_loss‚2.407860214992236X‚ªM Ž„ _step‚515 0loss/FRt2/assignment_loss‚3.2392524416728685 1loss/FBt2/assignment_loss‚0.09460819296337471 1loss/SRqqt1/detection_loss‚1.0394227783588215 .loss/FBt1/detection_loss‚2.615449649592241 _runtime‚2720.661390009 2loss/SRqqt1/assignment_loss‚1.3820629994238705 epoch‚0 $ _timestamp‚1.785812975471623e+09 2loss/SRqqt2/assignment_loss‚1.2383751947066335 2loss/FBt1/assignment_loss‚0.005418609707703581 .loss/FRt2/detection_loss‚2.407860214992236 1loss/SRqqt2/detection_loss‚1.9814287840587195 trainer/global_step‚6449 0loss/FRt1/assignment_loss‚3.4638357812708076 /loss/FRt1/detection_loss‚2.1628279713067142 .loss/FBt2/detection_loss‚1.253425591139655 &loss/total_loss‚20.883968209193647ƒwÞü‘‚(:† ø¯ÅÓô«¬ƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚41.0 gpu.0.powerWatts‚129.246' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.082 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦>ºÃn‚(:ã  ø¯ÅÓÂÕƒ disk.rbd1.in‚0 disk.drbd1828.in‚0 disk.drbd1584.in‚0 disk.drbd1584.out‚0* proc.memory.percent‚17.172330617904663 network.recv‚422046 disk.drbd1938.in‚0 disk.sda.out‚ 87.98828125 disk.drbd1510.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1356.in‚0% memory_percent‚41.237080097198486) disk./.usagePercent‚32.79228026612001 disk.rbd0.in‚0 disk.drbd1285.in‚0) proc.memory.availableMB‚ 38510.8671875 disk.dm-0.in‚ 4.9296875 disk.drbd1510.out‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd1828.out‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 87.98828125 disk.drbd1356.out‚0 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd1673.in‚0 disk.drbd1285.out‚0 disk.drbd1550.in‚0 disk.nvme0n1.in‚ 4.9296875$ disk.nvme0n1.out‚625.76416015625 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.sda.in‚ 0.68359375$ disk./.usageGB‚586.3592758178711 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd2041.in‚0 proc.cpu.threads‚29 network.sent‚ 5.980676e+06 disk.drbd1817.out‚0 disk.rbd1.out‚ 11.0166015625 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.rbd0.out‚0$ proc.memory.rssMB‚11254.05859375 cpu‚13.778959082768013! disk.dm-0.out‚625.76416015625 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd2078.out‚0 disk.drbd1817.in‚0¬D·F2‘- _wandbruntime‚2741  _runtime‚2741?§tÎÉ’Ã ' lr-AdamW/pg1‚0.00016749343996800433 ' lr-AdamW/pg2‚0.00016749343996800433 trainer/global_step‚6499 % _timestamp‚1.7858129961064613e+09 _runtime‚2741.296398778 _step‚516‘ dÎ“È ' lr-AdamW/pg1‚0.00016749343996800433 ' lr-AdamW/pg2‚0.00016749343996800433 trainer/global_step‚6499 % _timestamp‚1.7858129961064613e+09 _runtime‚2741.296398778 _step‚516„0Ý™2”- _wandbruntime‚2741  _runtime‚2741JyZÒ'•ÌO YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚111588485192704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25592897 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚108815681796832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚111852083534848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚111588502232064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111830644 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚111852108438528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5873717745152 % _timestamp‚1.7858129961073751e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚68636909 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25592895 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18611354 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5719093957120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5873856807936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43184077 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5719181386752 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚68646565 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18611617 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚114520761889932 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚517 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚68646565 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5705182935584 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚117307579149824 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5719093957120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚68636909 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚117307579149824 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚114520864732416 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43184079 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚117725965246464 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _runtime‚2741.300634171 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43184079 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚117725801280000 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 111820986 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚108815664757472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚263 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 trainer/global_step‚6499 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚111588502232064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43184077 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚111852083534848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚111588502232064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111830644 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚111852108438528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5873717745152 % _timestamp‚1.7858129961073751e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚68636909 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25592895 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18611354 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5719093957120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5873856807936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43184077 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5719181386752 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚68646565 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18611617 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚114520761889932 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚517 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚68646565 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5705182935584 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚117307579149824 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5719093957120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚68636909 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚117307579149824 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚114520864732416 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43184079 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚117725965246464 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _runtime‚2741.300634171 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43184079 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚117725801280000 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 111820986 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚108815664757472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QIDeviceStatsMonitor.on_train_pƒ˜Ýbatch_start/inactive_split.small_pool.current‚263 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 trainer/global_step‚6499 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚111588502232064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43184077 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111847824 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5705974605682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚111605651955712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 111838152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18614354 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111847824 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚111869290651648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚108832404929792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚111869315555328 % _timestamp‚1.7858129964665592e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚68647432 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5874623610880 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5719973576704 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚68647432 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚117744074027520 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43190720 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚68657102 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43190720 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚117743914262528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚68657102 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _step‚518 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚265 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚114538482378070 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚108832421969152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44211186 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚114538379535474 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 111838152 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚68657102 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _step‚518 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚265 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚114538482378070 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚108832421969152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44211186 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚114538379535474 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 111838152 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111847824 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5705974605682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚111605651955712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 111838152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18614354 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111847824 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚111869290651648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚108832404929792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚111869315555328 % _timestamp‚1.7858129964665592e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚68647432 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5874623610880 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5719973576704 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚68647432 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚117744074027520 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43190720 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚68657102 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43190720 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚117743914262528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0†Æ^É­2š- _wandbruntime‚2741  _runtime‚2741R¬ö0›ý 0loss/FRt1/assignment_loss‚3.3461291254321233 1loss/SRqqt2/assignment_loss‚1.337255475170752 /loss/FBt1/detection_loss‚2.1918474435806274 0loss/FRt2/assignment_loss‚3.3487350452377136 0loss/FBt1/assignment_loss‚0.5753904323966708 1loss/SRqqt1/detection_loss‚0.8608071558196477 &loss/total_loss‚21.087892974724923 epoch‚0 # _timestamp‚1.78581299646776e+09 _runtime‚2741.657548376 2loss/SRqqt1/assignment_loss‚1.2239441793589365 .loss/FRt1/detection_loss‚2.656847083304502 1loss/SRqqt2/detection_loss‚2.0030166853536318 2loss/FBt2/assignment_loss‚0.007425066024753828 .loss/FRt2/detection_loss‚2.422271184174411 /loss/FBt2/detection_loss‚1.1142240988711516 trainer/global_step‚6499 _step‚519µŒ×,œ‚ 2loss/FBt2/assignment_loss‚0.007425066024753828 .loss/FRt2/detection_loss‚2.422271184174411 /loss/FBt2/detection_loss‚1.1142240988711516 trainer/global_step‚6499 _step‚519 0loss/FRt1/assignment_loss‚3.3461291254321233 1loss/SRqqt2/assignment_loss‚1.337255475170752 /loss/FBt1/detection_loss‚2.1918474435806274 0loss/FRt2/assignment_loss‚3.3487350452377136 0loss/FBt1/assignment_loss‚0.5753904323966708 1loss/SRqqt1/detection_loss‚0.8608071558196477 &loss/total_loss‚21.087892974724923 epoch‚0 # _timestamp‚1.78581299646776e+09 _runtime‚2741.657548376 2loss/SRqqt1/assignment_loss‚1.2239441793589365 .loss/FRt1/detection_loss‚2.656847083304502 1loss/SRqqt2/detection_loss‚2.0030166853536318‡§ý;›‚(: ‡°ÅÓ°Ê¢ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚129.05' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.01666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0–em“fž‚(:Û  ‡°ÅÓ±ØÚƒ disk.drbd1965.in‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0 network.sent‚ 6.02572e+06 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1510.out‚0 cpu‚13.778136217523922 proc.cpu.threads‚29 disk.rbd1.in‚0 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.drbd2111.out‚0 disk.drbd1552.out‚0! disk.nvme0n1.out‚ 626.67578125 disk.drbd1584.out‚0 disk.drbd1285.out‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.68359375 disk.dm-0.in‚ 4.9296875 disk.drbd1817.in‚0 network.recv‚424418 disk.sda2.in‚ 0.68359375 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1817.out‚0 disk.drbd1510.in‚0 disk.drbd1673.out‚0 disk.rbd1.out‚ 11.1103515625 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.sda1.in‚0$ memory_percent‚41.23750925064087) proc.memory.availableMB‚ 38510.5859375* proc.memory.percent‚17.172330617904663 disk.sda2.out‚ 88.43359375 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd1552.in‚0 disk.drbd1938.in‚0$ disk./.usageGB‚586.3592414855957 disk.drbd1584.in‚0 disk.drbd1965.out‚0 disk.drbd1828.out‚0 disk.drbd2118.in‚0 disk.drbd1356.out‚0 disk.sda.out‚ 88.43359375 disk.drbd1289.in‚0 disk.sda1.out‚0$ proc.memory.rssMB‚11254.05859375 disk.nvme0n1.in‚ 4.9296875 disk.drbd1289.out‚0 disk.drbd2078.in‚0 disk.drbd2078.out‚0) disk./.usagePercent‚32.79227834607944 disk.drbd1550.out‚0 disk.dm-0.out‚ 626.67578125 disk.drbd2111.in‚0 disk.drbd1938.out‚0Ù'dŸ‚(:’ –°ÅÓ…’•ƒ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚130.696' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.565333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ] k ‚(:à  –°ÅÓôñÙƒ disk.drbd2111.in‚0 network.sent‚ 6.0708e+06$ disk.nvme0n1.out‚631.70166015625 disk.drbd1965.out‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0$ memory_percent‚41.23753309249878 disk.drbd1584.out‚0 disk.drbd2111.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 89.98828125) proc.memory.availableMB‚ 38510.5703125 disk.drbd2041.in‚0 disk.drbd1673.in‚0 cpu‚13.777632553910626 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.dm-0.in‚ 4.9296875 disk.sda1.out‚0* proc.memory.percent‚17.172330617904663 proc.cpu.threads‚29 disk.drbd1938.out‚0$ disk./.usageGB‚586.3601722717285 disk.drbd2041.out‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.drbd1931.in‚0 disk.sda.out‚ 89.98828125 disk.drbd1828.in‚0 disk.drbd1552.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2078.out‚0 disk.drbd1356.in‚0 disk.drbd1550.in‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚ 4.9296875 disk.drbd1510.in‚0 disk.drbd2118.out‚0 disk.drbd1226.in‚0! disk.dm-0.out‚631.70166015625 disk.drbd2118.in‚0 disk.drbd1673.out‚0 disk.rbd1.out‚ 11.1728515625 disk.drbd1931.out‚0 disk.drbd1552.in‚0 disk.drbd1550.out‚0 network.recv‚426767 disk.rbd1.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.sda1.in‚0) disk./.usagePercent‚32.79233040051261•Eôâ2¡- _wandbruntime‚2761  _runtime‚2761†0`=ɢà ' lr-AdamW/pg1‚0.00016878204929226962 ' lr-AdamW/pg2‚0.00016878204929226962 trainer/global_step‚6549 % _timestamp‚1.7858130165250347e+09 _runtime‚2761.714996664 _step‚520Duœ{Î£È ' lr-AdamW/pg2‚0.00016878204929226962 trainer/global_step‚6549 % _timestamp‚1.7858130165250347e+09 _runtime‚2761.714996664 _step‚520 ' lr-AdamW/pg1‚0.00016878204929226962ˆ Ü_=2¤- _wandbruntime‚2761  _runtime‚2761sÏß׃ ¥ÌO O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25789747 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚6549 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43516229 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚69174567 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚109652673373472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚69164911 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43516227 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚239 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5748985595968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43516229 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚69164911 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5763090092544 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.þ¤×õOpeak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚118631536242688 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚118209913435648 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚112446823343104 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚112446823343104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚118210017904640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5763090092544 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚112446840382464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25789745 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18756245 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚109652690412832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚112446840382464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚521 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43516227 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚112712439374848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚112712464278528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121S><×'¦ÑO KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5763177522176 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44546231 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18756484 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚112446840382464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚521 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43516227 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚112712439374848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚112712464278528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25789747 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚6549 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43516229 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚69174567 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚109652673373472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚69164911 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43516227 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚239 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5748985595968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43516229 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚69164911 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5763090092544 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚118631536242688 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚118209913435648 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚112446823343104 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚112446823343104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚118210017904640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5763090092544 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚112446840382464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25789745 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18756245 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚109652690412832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5763969712128 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚69175434 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43522872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5749863069190 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25793682 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚109669413545792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112707976 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44553046 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚112464007145472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43522870 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 % _timestamp‚1.7858130168863401e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112707976 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112698304 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚118649490216960 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚112729671395328 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚112463990106112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚115419276614982 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44552783 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚112729646491648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18759362 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5919843725312 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5749948872426 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25793684 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚118228064294400 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5764057148928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚2762.076636978 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43522870 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚118228064294400 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18759101 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚118228064294400 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5764057148928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚2762.076636978 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43522870 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚118228064294400 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18759101 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5763969712128 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚69175434 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43522872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5749863069190 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25793682 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚109669413545792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112707976 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44553046 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚112464007145472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43522870 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 % _timestamp‚1.7858130168863401e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112707976 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112698304 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚118649490216960 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚112729671395328 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚112463990106112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚115419276614982 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44552783 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚112729646491648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18759362ŠC*?2ª- _wandbruntime‚2762  _runtime‚2762çd~ë«ý 1loss/SRqqt1/detection_loss‚1.0138902194463673 .loss/FBt1/detection_loss‚2.083740584552288 2loss/SRqqt2/assignment_loss‚1.0454080969822117 .loss/FBt2/detection_loss‚1.433086725184694 $loss/total_loss‚20.8558601647348 epoch‚0 trainer/global_step‚6549 % _timestamp‚1.7858130168875587e+09 0loss/FRt1/assignment_loss‚3.0027751195430756 1loss/FBt1/assignment_loss‚0.47310483210048915 /loss/FRt1/detection_loss‚2.1704063856601716 /loss/FRt2/detection_loss‚2.6573293819313957 1loss/SRqqt2/detection_loss‚2.3651521760152607 /loss/FRt2/assignment_loss‚3.405239639537675 2loss/SRqqt1/assignment_loss‚1.1344797766934602 1loss/FBt2/assignment_loss‚0.07124722708770803 _runtime‚2762.077339422 _step‚523 ʬ‚ 2loss/SRqqt2/assignment_loss‚1.0454080969822117 .loss/FBt2/detection_loss‚1.433086725184694 $loss/total_loss‚20.8558601647348 epoch‚0 trainer/global_step‚6549 % _timestamp‚1.7858130168875587e+09 0loss/FRt1/assignment_loss‚3.0027751195430756 1loss/FBt1/assignment_loss‚0.47310483210048915 /loss/FRt1/detection_loss‚2.1704063856601716 /loss/FRt2/detection_loss‚2.6573293819313957 1loss/SRqqt2/detection_loss‚2.3651521760152607 /loss/FRt2/assignment_loss‚3.405239639537675 2loss/SRqqt1/assignment_loss‚1.1344797766934602 1loss/FBt2/assignment_loss‚0.07124722708770803 _runtime‚2762.077339422 _step‚523 1loss/SRqqt1/detection_loss‚1.0138902194463673 .loss/FBt1/detection_loss‚2.083740584552288‹˜6çk›­‚(: ¥°ÅÓߟœƒ gpu.0.gpu‚34.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚97.891' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.63033333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Sâtÿh®‚(:Ý  ¥°ÅÓñ«äƒ disk.drbd1817.out‚0 disk.sda.in‚ 0.68359375 proc.cpu.threads‚29 disk.drbd2041.in‚0 disk.rbd0.in‚0 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0" disk.nvme0n1.out‚ 639.017578125 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd2118.in‚0 disk.drbd1931.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚ 4.9296875 disk.dm-0.out‚ 639.017578125 disk.drbd1289.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1965.out‚0 disk.sda1.out‚0 disk.drbd1552.in‚0* proc.memory.percent‚17.172330617904663 disk.rbd0.out‚0* proc.memory.availableMB‚38510.50390625 disk.drbd1356.out‚0 disk.drbd1550.out‚0 disk.drbd1285.out‚0 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0# memory_percent‚41.2376344203949 disk.drbd2111.out‚0 disk.drbd1552.out‚0 disk.sda2.out‚ 92.98828125 disk.nvme0n1.in‚ 4.9296875 disk.drbd1673.in‚0 disk.drbd2118.out‚0 cpu‚13.777045230155446 network.recv‚427664 disk.drbd1673.out‚0 disk.rbd1.out‚ 11.1728515625 disk.drbd1356.in‚0 disk.sda.out‚ 92.98828125 disk.drbd1226.in‚0 disk.drbd1828.in‚0) disk./.usagePercent‚32.79210362238779 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1965.in‚0 disk.drbd1510.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1817.in‚0 disk.drbd1931.out‚0 disk.drbd1285.in‚0 disk.sda1.in‚0 network.sent‚ 6.07384e+06$ disk./.usageGB‚586.3561172485352 disk.drbd2041.out‚0-ûÍ­2¯- _wandbruntime‚2782  _runtime‚2782—>œÉ°Ã ' lr-AdamW/pg1‚0.00017007065861653493 ' lr-AdamW/pg2‚0.00017007065861653493 trainer/global_step‚6599 % _timestamp‚1.7858130377229145e+09 _runtime‚2782.912859314 _step‚524DÄ­Î±È trainer/global_step‚6599 % _timestamp‚1.7858130377229145e+09 _runtime‚2782.912859314 _step‚524 ' lr-AdamW/pg1‚0.00017007065861653493 ' lr-AdamW/pg2‚0.00017007065861653493ŒÒÓ>K2²- _wandbruntime‚2782  _runtime‚2782þvhFÒ'³ÌO E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚113572820118528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚119112352190464 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5807173657600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5807086227968 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43848377 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚116282658891432 trainer/global_step‚6599 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44887452 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚113572795214848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚113305178532864 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚113305161493504 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858130377238357e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚69702569 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚119112247721472 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43848379 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 113550948 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5964095431168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚113305161493504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _runtime‚2782.917656117 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5792874059476 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚119112247721472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5807173657600 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25986597 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43848377 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5964236591104 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚116282556048948 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25986595 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚69692913 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18900595 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚110489681989472 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚525 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚113305178532864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18900855 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260ßJù×'´ÑO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚113305178532864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18900855 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚113572820118528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚119112352190464 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5807173657600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5807086227968 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43848377 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚116282658891432 trainer/global_step‚6599 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44887452 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚113572795214848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚113305178532864 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚113305161493504 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 % _timestamp‚1.7858130377238357e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚69702569 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚119112247721472 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43848379 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 113550948 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5964095431168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚113305161493504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _runtime‚2782.917656117 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5792874059476 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚119112247721472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5807173657600 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25986597 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43848377 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5964236591104 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚116282556048948 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25986595 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚69692913 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18900595 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚110489681989472 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚525 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538ù4C2µ- _wandbruntime‚2783  _runtime‚27839þ}{P ¶ÖM L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 113568128 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚119555157331456 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 113558456 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25990534 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚6599 % _timestamp‚1.7858130380787609e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚113590002331648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚69713106 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚69703436 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5964995234816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43855022 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚113322328256512 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43855022 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocatedÊ" Œ‚119130398580224 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43855020 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44894161 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚113322328256512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18903394 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚119554997566464 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚110506422161792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚233 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚113322345295872 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44893926 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5793837335934 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚69713106 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5807965847552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚235 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5808053284352 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚113590027235328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚110506439201152 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚119130294104064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18903627 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5965130096128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚116300173694490 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43855020 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5808053284352 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 113558456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚119130398580224 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚116300276537086 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚69703436 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0!Q/á&·ÛM JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 113568128 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43855020 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44894161 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚113322328256512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18903394 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚119554997566464 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚110506422161792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚233 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚113322345295872 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44893926 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5793837335934 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚69713106 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5807965847552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚235 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5808053284352 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚113590027235328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚110506439201152 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚119130294104064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18903627 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5965130096128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚116300173694490 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43855020 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5808053284352 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 113558456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚119130398580224 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚116300276537086 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚69703436 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 113568128 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚119555157331456 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 113558456 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25990534 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚6599 % _timestamp‚1.7858130380787609e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚113590002331648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚69713106 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚69703436 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5964995234816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43855022 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚113322328256512 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43855022 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚119130398580224 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E( º… /loss/FRt1/detection_loss‚2.2795086451048077 1loss/SRqqt2/detection_loss‚1.9889190384585942 /loss/FBt1/detection_loss‚2.1382495967671273 trainer/global_step‚6599 _step‚527 /loss/FRt2/detection_loss‚2.5866867878607342 /loss/FBt2/detection_loss‚1.3326481746775762 _runtime‚2783.269776056 2loss/SRqqt1/assignment_loss‚1.2555577648679415 1loss/FBt1/assignment_loss‚0.08111519790254533 1loss/FBt2/assignment_loss‚0.06835129376303521 1loss/SRqqt1/detection_loss‚0.9704948702578743 &loss/total_loss‚20.986518915214923 epoch‚0 % _timestamp‚1.7858130380799713e+09 /loss/FRt1/assignment_loss‚3.325091446554938 /loss/FRt2/assignment_loss‚3.239805911978086 2loss/SRqqt2/assignment_loss‚1.7200901870216643ndM¤œ»‚(:‘ ´°ÅÓ×რgpu.0.gpu‚36.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚127.837' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.61233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÍÇ.f¼‚(:Û  ´°ÅÓžÎ̃ disk.drbd1356.in‚0 disk.drbd2041.out‚0 disk.sda.out‚ 95.51953125 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚ 9.50390625 cpu‚13.776247743546467 disk.drbd1510.out‚0 disk.rbd1.out‚ 11.205078125# memory_percent‚41.2375271320343 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.sda1.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd1356.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.drbd1965.in‚0 disk.drbd2118.in‚0$ disk./.usageGB‚586.3652877807617 disk.drbd2041.in‚0 disk.drbd1510.in‚0 disk.nvme0n1.out‚ 641.5390625 disk.drbd1828.out‚0 disk.dm-0.out‚ 641.5390625 disk.drbd1817.in‚0 network.sent‚ 6.119029e+06 disk.rbd0.in‚0 disk.drbd1584.in‚0 disk.drbd1828.in‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1285.out‚0 disk.drbd1938.out‚0 disk.drbd1931.out‚0 disk.drbd1550.in‚0 disk.dm-0.in‚ 9.50390625 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 network.recv‚430303 disk.drbd1226.in‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1552.out‚0 disk.drbd1226.out‚0 disk.drbd1289.out‚0 disk.sda2.in‚ 0.68359375 proc.cpu.threads‚29 disk.drbd2078.out‚0 disk.sda1.in‚0* proc.memory.availableMB‚38510.57421875) disk./.usagePercent‚32.79261648655719 disk.drbd1584.out‚0 disk.drbd1817.out‚0 disk.sda2.out‚ 95.51953125 disk.rbd0.out‚0 disk.drbd1938.in‚0ä¨>Ø2½- _wandbruntime‚2803  _runtime‚2803±ÕµÊɾà ' lr-AdamW/pg1‚0.00017135926794080024 ' lr-AdamW/pg2‚0.00017135926794080024 trainer/global_step‚6649 % _timestamp‚1.7858130584862263e+09 _runtime‚2803.676160703 _step‚528¹µàÎ¿È % _timestamp‚1.7858130584862263e+09 _runtime‚2803.676160703 _step‚528 ' lr-AdamW/pg1‚0.00017135926794080024 ' lr-AdamW/pg2‚0.00017135926794080024 trainer/global_step‚6649—E¯Û2À- _wandbruntime‚2803  _runtime‚2803ÁaêÐÒ'ÁÌO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19043334 _step‚529 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚120014582007296 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _runtime‚2803.680639317 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚114433175958528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚45227032 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚120442472989696 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5836762522984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚70220915 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚120014686476288 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 114401442 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6009321934848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚70230571 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚253 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44180527 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚114163499643904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26183447 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5851169793024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114411100 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚117163555970940 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚114433151054848 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚111326707644832 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5851082363392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚70230571 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚114163516683264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120014582007296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚70220915 % _timestamp‚1.7858130584872649e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44180529 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5851082363392 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚117163453128456 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136…jR‘&ÂÑO N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44180527 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚114163516683264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44180529 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚120442628567552 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚111326690605472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5836848326108 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚120014686476288 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6009452609024 H•àî±!BDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5851169793024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚114163499643904 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26183445 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19043585 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚114433151054848 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚111326707644832 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5851082363392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚70230571 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚114163516683264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120014582007296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚70220915 % _timestamp‚1.7858130584872649e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44180529 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5851082363392 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚117163453128456 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19043334 _step‚529 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚120014582007296 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _runtime‚2803.680639317 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚114433175958528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚45227032 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚120442472989696 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5836762522984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚70220915 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚120014686476288 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 114401442 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6009321934848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚70230571 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚253 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44180527 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚114163499643904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26183447 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5851169793024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114411100 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚117163555970940 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114428280 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114418608 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5837639996206 trainer/global_step‚6649 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5837725799442 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44187172 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5851961982976 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44187170 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120032628389888 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45233549 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19046167 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚111343430777792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114428280 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚114180683446272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚114180666406912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44187172 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6010358492160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5852049419776 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚117181070773998 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚70241108 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5851961982976 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5852049419776 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19046436 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚120460741567488 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚114450383075328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚117181173616594 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚120032732866048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚120460581802496 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚114450358171648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 _runtime‚2804.032206381 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚70231438 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚70241108 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26187384 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚269 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0jÑ €à&ÅÚM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44187172 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5851961982976 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44187170 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120032628389888 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45233549 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19046167 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚111343430777792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114428280 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚114180683446272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚114180666406912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44187172 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6010358492160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5852049419776 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚117181070773998 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚70241108 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5851961982976 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5852049419776 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19046436 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚120460741567488 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚114450383075328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚117181173616594 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚120032732866048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚120460581802496 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚114450358171648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 _runtime‚2804.032206381 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚70231438 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚70241108 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26187384 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚269 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _step‚530 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚114180666406912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114428280 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114418608 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5837639996206 trainer/global_step‚6649 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5837725799442 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0’O˜½12Æ- _wandbruntime‚2804  _runtime‚2804ÿ%ÿÓÇý 1loss/FBt1/assignment_loss‚0.41098219125997276 1loss/FBt2/assignment_loss‚0.05180681678738982 1loss/SRqqt1/detection_loss‚1.0341380502699928 % _timestamp‚1.7858130588431275e+09 1loss/SRqqt1/assignment_loss‚1.152490407781785 /loss/FRt1/detection_loss‚2.5156306380575355 /loss/FBt1/detection_loss‚2.0904478430747986 epoch‚0 1loss/SRqqt2/assignment_loss‚1.393623635937006 %loss/total_loss‚20.86500304719148 trainer/global_step‚6649 _step‚531 /loss/FRt1/assignment_loss‚3.188787034579686 /loss/FRt2/detection_loss‚2.5417172506798145 0loss/SRqqt2/detection_loss‚2.107022179529453 /loss/FBt2/detection_loss‚1.3681349468151374 _runtime‚2804.032923292 0loss/FRt2/assignment_loss‚3.0102220524189085mÞ“°È‚ trainer/global_step‚6649 _step‚531 /loss/FRt1/assignment_loss‚3.188787034579686 /loss/FRt2/detection_loss‚2.5417172506798145 0loss/SRqqt2/detection_loss‚2.107022179529453 /loss/FBt2/detection_loss‚1.3681349468151374 _runtime‚2804.032923292 0loss/FRt2/assignment_loss‚3.0102220524189085 1loss/FBt1/assignment_loss‚0.41098219125997276 1loss/FBt2/assignment_loss‚0.05180681678738982 1loss/SRqqt1/detection_loss‚1.0341380502699928 % _timestamp‚1.7858130588431275e+09 1loss/SRqqt1/assignment_loss‚1.152490407781785 /loss/FRt1/detection_loss‚2.5156306380575355 /loss/FBt1/detection_loss‚2.0904478430747986 epoch‚0 1loss/SRqqt2/assignment_loss‚1.393623635937006 %loss/total_loss‚20.86500304719148“[‰)œÉ‚(:‘ ðÅÓÒ݇ƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚131.203' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.73433333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0~hÖ\Ê‚(:Ñ  ðÅÓ߰΃ disk.drbd1510.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.drbd1552.in‚0 disk.rbd1.in‚0 proc.cpu.threads‚29 disk.rbd0.out‚0 disk.drbd1289.out‚0' disk./.usagePercent‚32.792630140179 disk.drbd1289.in‚0 disk.nvme0n1.in‚9.59375 disk.rbd0.in‚0 disk.dm-0.in‚9.59375 disk.drbd1931.out‚0 disk.drbd1965.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2078.in‚0 disk.drbd2111.out‚0 disk.drbd1938.out‚0 disk.sda.out‚95.9375 disk.drbd1550.out‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd1931.in‚0 disk.drbd1817.out‚0 cpu‚13.775718081203955 disk.drbd1584.in‚0 disk.drbd1226.out‚0 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0 disk.drbd2118.out‚0$ disk./.usageGB‚586.3655319213867 disk.drbd2111.in‚0 disk.sda2.in‚ 0.68359375 disk.rbd1.out‚ 11.384765625 disk.drbd1356.out‚0# disk.nvme0n1.out‚644.9150390625 disk.drbd1285.out‚0 disk.dm-0.out‚644.9150390625 disk.sda1.out‚0 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.sda1.in‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0 disk.drbd2078.out‚0 disk.sda2.out‚95.9375* proc.memory.percent‚17.172330617904663 network.sent‚ 6.164075e+06 network.recv‚432579 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.drbd1965.out‚0$ memory_percent‚41.23796224594116) proc.memory.availableMB‚ 38510.2890625 disk.drbd1285.in‚0 ¡ª›ŽË‚(:ƒ Ò°ÅÓÝ­ƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚97.92' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.64 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0a• ÚRÌ‚(:Ý  Ò°ÅÓýÖƒ disk.drbd1550.out‚0 disk.rbd1.out‚ 11.4472656251O«› disk.drbd2041.in‚0 disk.sda.out‚ 96.21484375 disk.drbd1965.in‚0 network.recv‚435003 disk.rbd1.in‚0 disk.drbd1226.out‚0 cpu‚13.775236816281813 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1510.out‚0$ proc.memory.rssMB‚11254.05859375$ disk./.usageGB‚586.3605995178223 disk.drbd1673.out‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 96.21484375 disk.drbd2078.in‚0) proc.memory.availableMB‚ 38510.2890625 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.sda1.in‚0) disk./.usagePercent‚32.79235429435079! disk.dm-0.out‚647.82666015625 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.drbd2041.out‚0 disk.drbd2078.out‚0* proc.memory.percent‚17.172330617904663 disk.nvme0n1.in‚9.59375$ disk.nvme0n1.out‚647.82666015625 disk.drbd2111.out‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1584.in‚0 disk.dm-0.in‚9.59375 disk.drbd1817.out‚0 disk.drbd1965.out‚0 network.sent‚ 6.209151e+06 disk.drbd1938.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd2118.in‚0 disk.drbd1931.in‚0 proc.cpu.threads‚29 disk.drbd1817.in‚0$ memory_percent‚41.23796224594116 disk.drbd1356.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1584.out‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.sda1.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1226.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0Úá1 2Í- _wandbruntime‚2824  _runtime‚2824ž…¡¬ÉÎà % _timestamp‚1.7858130792712138e+09 _runtime‚2824.461151683 _step‚532 ' lr-AdamW/pg1‚0.00017264787726506555 ' lr-AdamW/pg2‚0.00017264787726506555 trainer/global_step‚6699ÉHÆÀÎÏÈ _runtime‚2824.461151683 _step‚532 ' lr-AdamW/pg1‚0.00017264787726506555 ' lr-AdamW/pg2‚0.00017264787726506555 trainer/global_step‚6699 % _timestamp‚1.7858130792712138e+09”%ÉÂê2Ð- _wandbruntime‚2824  _runtime‚28246©<ÎÑ'ÑËO YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚115021837794304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚120917020762112 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26380297 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 $ _timestamp‚1.785813079272147e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44512679 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120916916293120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5895165928448 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44512677 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚2824.465628918 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚45567361 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26380295 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 115261594 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚121348043954688 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚112163716260832 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚115021854833664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚112163699221472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44512677 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5895165928448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚115293531798528 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120916916293120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5895165928448 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44512677 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚2824.465628918 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚45567361 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26380295 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 115261594 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚121348043954688 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚112163716260832 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360•Q¾[ 2Ó- _wandbruntime‚2824  _runtime‚2824ºpá,Ü&ÔÖM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚120935067151872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚115310714011648 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 _step‚534 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚70769110 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚115039021596672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44519320 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚121366310535168 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚70759440 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120934962675712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚115310738915328 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚70759440 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45573946 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26384234 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5895958118400 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44519322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115039004557312 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115288432 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚112180456433152 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5896045555200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5881528459714 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6055571619840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115278760 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19189714 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚121366161255936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115039004557312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚70769110 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚6699 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5881614262950 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115039021596672 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19189969 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.õ[Fí%allocated‚ 6055571619840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115278760 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19189714 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚121366161255936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115039004557312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚70769110 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚6699 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5881614262950 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115039021596672 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19189969 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120934962675712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚115310738915328 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚70759440 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45573946 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26384234 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5895958118400 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44519322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115039004557312 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115288432 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚112180456433152 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5896045555200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5881528459714 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5939074634240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚116153887638528 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚121819250578944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚113000707837472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5924625253124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚266 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 $ _timestamp‚1.785813100345031e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19329756 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44844829 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26577147 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚122253579372032 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 116121746 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5939162063872 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚71276919 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚122253730755584 trainer/global_step‚6749 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚121819355047936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5939074634240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44844829 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45906901 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44844827 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚268 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5924539450000 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚71286575 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5939162063872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26577145 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚113000724876832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚537 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚115880192984064 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0-O M¯!àÐO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚268 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5924539450000 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚71286575 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5939162063872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26577145 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚113000724876832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚537 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚115880192984064 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚118925247287472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚71276919 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5939074634240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚116153887638528 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚121819250578944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚113000707837472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5924625253124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚266 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 $ _timestamp‚1.785813100345031e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19329756 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44844829 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26577147 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚122253579372032 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 116121746 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5939162063872 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚71276919 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚122253730755584 trainer/global_step‚6749 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚121819ÁȧÇ'355047936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5939074634240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44844829 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45906901 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44844827 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816™`-,¥2á- _wandbruntime‚2845  _runtime‚2845”¢bTÜ&âÖM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45913925 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5925502726458 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5940041690624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5939954253824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116148584 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19332841 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44851470 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚538 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115897342707712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44851472 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚71297112 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚6749 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚121837401437696 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚113017465049152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚122271843436032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116138912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858131006948133e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44851470 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚121837296961536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5939954253824 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚71287442 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚116171094755328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚122271683671040 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6100613819392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚71297112 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚118942967775610 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚118942864933014 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚121837401437696 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116148584 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116138912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚113017448009792 _runtime‚2845.885095246 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45913673 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6100748680704 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5925416923222 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670ÆaDá&ãÛM =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5939954253824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116148584 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19332841 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44851470 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚538 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115897342707712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44851472 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚71297112 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚6749 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚121837401437696 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚113017465049152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚122271843436032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116138912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858131006948133e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44851470 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚121837296961536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5939954253824 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚71287442 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚116171094755328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚122271683671040 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6100613819392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚71297112 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚118942967775610 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚118942864933014 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚121837401437696 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116148584 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116138912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚113017448009792 _runtime‚2845.885095246 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45913673 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6100748680704 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5925416923222 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45913925 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5925502726458 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5940041690624šü´‡z2ä- _wandbruntime‚2845  _runtime‚2845ÒQmåþ 0loss/FRt2/assignment_loss‚3.2601477740562124 1loss/SRqqt1/detection_loss‚1.0397710175681356 /loss/FBt1/detection_loss‚1.5581390644822801 0loss/FRt1/assignment_loss‚2.9407949454562607 1loss/SRqqt1/assignment_loss‚1.124000612765903 2loss/FBt2/assignment_loss‚0.008356503080548805 1loss/SRqqt2/detection_loss‚2.1594739319422307 /loss/FBt2/detection_loss‚1.2754282411611215 epoch‚0 % _timestamp‚1.7858131006960151e+09 _runtime‚2845.885803876 .loss/FRt1/detection_loss‚2.349497909809268 .loss/FRt2/detection_loss‚2.753700453941136 &loss/total_loss‚20.351894882798447 _step‚539 1loss/SRqqt2/assignment_loss‚1.203852811876016 0loss/FBt1/assignment_loss‚0.6787316166593362 trainer/global_step‚6749ÄŒ= æƒ 1loss/SRqqt2/assignment_loss‚1.203852811876016 0loss/FBt1/assignment_loss‚0.6787316166593362 trainer/global_step‚6749 0loss/FRt2/assignment_loss‚3.2601477740562124 1loss/SRqqt1/detection_loss‚1.0397710175681356 /loss/FBt1/detection_loss‚1.5581390644822801 0loss/FRt1/assignment_loss‚2.9407949454562607 1loss/SRqqt1/assignment_loss‚1.124000612765903 2loss/FBt2/assignment_loss‚0.008356503080548805 1loss/SRqqt2/detection_loss‚2.1594739319422307 /loss/FBt2/detection_loss‚1.2754282411611215 epoch‚0 % _timestamp‚1.7858131006960151e+09 _runtime‚2845.885803876 .loss/FRt1/detection_loss‚2.349497909809268 .loss/FRt2/detection_loss‚2.753700453941136 &loss/total_loss‚20.351894882798447 _step‚539›Æ%œç‚(:‘ ð°ÅÓŸ¼é‚ gpu.0.gpu‚20.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚124.288' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.42933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0}™˜fè‚(:Û  ð°ÅÓÿ ­ƒ disk.nvme0n1.in‚9.59375 proc.cpu.threads‚29$ disk./.usageGB‚586.3606567382812 disk.drbd1584.in‚0 disk.drbd1550.in‚0 disk.drbd1285.out‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.rbd1.out‚ 11.4794921875$ proc.memory.rssMB‚11254.05859375 disk.drbd1510.in‚0 disk.drbd1226.in‚0 network.recv‚438457 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.drbd1673.out‚0 disk.drbd2118.out‚0( disk./.usagePercent‚32.7923574944184 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd1817.in‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0 network.sent‚ 6.257261e+06 disk.dm-0.in‚9.59375 disk.drbd2111.out‚0 disk.sda.in‚ 0.68359375 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0# disk.nvme0n1.out‚654.6025390625 disk.drbd1356.in‚0 disk.drbd2041.out‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd2041.in‚0 disk.drbd1510.out‚0 cpu‚13.77386442196744 disk.drbd2078.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.sda2.out‚ 99.23828125 disk.rbd1.in‚0 disk.sda.out‚ 99.23828125 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd1356.out‚0* proc.memory.availableMB‚38510.43359375 disk.drbd1965.in‚0 disk.sda2.in‚ 0.68359375* proc.memory.percent‚17.172330617904663 disk.rbd0.out‚0 disk.dm-0.out‚654.6025390625$ memory_percent‚41.23774170875549-äâ1dé‚(:Ù  ÿ°ÅÓîÊă disk.sda.in‚ 0.68359375 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd1673.out‚0 disk.nvme0n1.in‚9.59375 disk.drbd1285.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd2111.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1817.in‚0 disk.drbd1584.in‚0! disk.dm-0.out‚657.00439453125 disk.drbd1289.in‚0 disk.rbd1.out‚ 11.6591796875 disk.drbd1828.in‚0 disk.drbd1931.out‚0 disk.drbd1828.out‚0$ disk./.usageGB‚586.3606567382812$ disk.nvme0n1.out‚657.00439453125 disk.drbd1552.out‚0 disk.drbd1673.in‚0 disk.drbd1226.in‚0 disk.drbd1289.out‚0 cpu‚13.77352692567736 disk.drbd1356.in‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.drbd1584.out‚0 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd2078.in‚0 network.sent‚ 6.302308e+06( disk./.usagePercent‚32.7923574944184 disk.drbd1817.out‚0 disk.drbd1931.in‚0 disk.sda1.in‚0 disk.drbd1552.in‚0$ memory_percent‚41.23774170875549 disk.drbd2041.out‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.dm-0.in‚9.59375 disk.drbd2118.out‚0 disk.sda.out‚ 99.671875 disk.drbd2111.out‚0 proc.cpu.threads‚29 disk.drbd2041.in‚0 network.recv‚440837 disk.drbd1938.in‚0 disk.sda2.out‚ 99.671875 disk.drbd1510.in‚0* proc.memory.availableMB‚38510.43359375$ proc.memory.rssMB‚11254.05859375 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd2118.in‚0 disk.drbd1938.out‚0"X]šê‚(: ÿ°ÅÓê–ð„ gpu.0.gpu‚37.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚131.2' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.73333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0–gX!2ë- _wandbruntime‚2866  _runtime‚2866Px¬Éìà % _timestamp‚1.7858131208405447e+09 _runtime‚2866.030503205 _step‚540 ' lr-AdamW/pg1‚0.00017522509591359615 ' lr-AdamW/pg2‚0.00017522509591359615 trainer/global_step‚6799…¹‰“ÎíÈ % _timestamp‚1.7858131208405447e+09 _runtime‚2866.030503205 _step‚540 ' lr-AdamW/pg1‚0.00017522509591359615 ' lr-AdamW/pg2‚0.00017522509591359615 trainer/global_step‚6799œ þóþ2î- _wandbruntime‚2866  _runtime‚2866…Ã}˜@ïËO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚122721689333760 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚113837716453472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46247270 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚71814577 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚113837733492832 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚122721584864768 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚116738531134464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46247536 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚119806144366980 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45176979 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚122721689333760 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26773997 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚123159256342528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5983158199296 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚266 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 116991556 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5983070769664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚123159109153280 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚71804921 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6145012864000 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712š"¥0Ö'ðÐO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5983158199296 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45176979 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26773995 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚116738531134464 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5983070769664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5968427913508 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5983070769664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚123159109153280 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚71804921 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6145012864000 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚122721689333760 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚113837716453472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46247270 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚71814577 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚113837733492832 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚122721584864768 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚116738531134464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46247536 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚119806144366980 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45176979 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚122721689333760 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26773997 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚123159256342528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5983158199296 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚266 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 116991556 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46254101 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117008736 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5969391189966 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚71815444 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚123177230426112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7858131211942756e+09 _runtime‚2866.384586197 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚116755697897472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19476425 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19476169 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116999064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚117031450595328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚113854456625792 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117008736 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚122739735723520 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45183620 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚122739735723520 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116999064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 trainer/global_step‚6799 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5983950389248 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45183622 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46254359 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚123177383899648 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚117031425691648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26777934 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5983950389248 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚113854473665152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45183622 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚116755680858112 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45183620 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚116755680858112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚71825114 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚116755680858112 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45183620 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚116755680858112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚71825114 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46254101 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117008736 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5969391189966 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚71815444 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚123177230426112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7858131211942756e+09 _runtime‚2866.384586197 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚116755697897472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19476425 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19476169 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116999064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚117031450595328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚113854456625792 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117008736 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚122739735723520 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45183620 >6DeviceStatsMonitor.onÝš¥Çy _train_batch_end/segment.all.peak‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚122739735723520 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116999064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 trainer/global_step‚6799 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5983950389248 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45183622 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46254359 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚123177383899648 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚117031425691648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0žj2ô- _wandbruntime‚2866  _runtime‚2866jxüNõþ 1loss/FBt1/assignment_loss‚0.23287010485964857 1loss/SRqqt1/detection_loss‚1.0592978220997435 % _timestamp‚1.7858131211954916e+09 _step‚543 2loss/SRqqt1/assignment_loss‚1.3692676397379149 2loss/SRqqt2/assignment_loss‚1.2037911175897247 .loss/FRt2/detection_loss‚2.443368742863337 .loss/FBt2/detection_loss‚1.219065189565299 &loss/total_loss‚21.235576948895297 trainer/global_step‚6799 /loss/FRt2/assignment_loss‚3.400634267595079 2loss/FBt2/assignment_loss‚0.035280658809186516 /loss/FRt1/detection_loss‚2.5275391743463627 1loss/SRqqt2/detection_loss‚2.3470042473391484 epoch‚0 _runtime‚2866.385286081 0loss/FRt1/assignment_loss‚3.1201957166194916 -loss/FBt1/detection_loss‚2.27726226747036RPS öƒ 2loss/FBt2/assignment_loss‚0.035280658809186516 /loss/FRt1/detection_loss‚2.5275391743463627 1loss/SRqqt2/detection_loss‚2.3470042473391484 epoch‚0 _runtime‚2866.385286081 0loss/FRt1/assignment_loss‚3.1201957166194916 -loss/FBt1/detection_loss‚2.27726226747036 1loss/FBt1/assignment_loss‚0.23287010485964857 1loss/SRqqt1/detection_loss‚1.0592978220997435 % _timestamp‚1.7858131211954916e+09 _step‚543 2loss/SRqqt1/assignment_loss‚1.3692676397379149 2loss/SRqqt2/assignment_loss‚1.2037911175897247 .loss/FRt2/detection_loss‚2.443368742863337 .loss/FBt2/detection_loss‚1.219065189565299 &loss/total_loss‚21.235576948895297 trainer/global_step‚6799 /loss/FRt2/assignment_loss‚3.400634267595079Ÿµ.*œ÷‚(:‘ Ž±ÅÓˆÁžƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚99.829' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚33.276333333333326 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0·Œ¸¿nø‚(:ã  Ž±ÅÓ’Òèƒ disk.drbd1510.out‚0 disk.sda2.out‚ 100.16015625 disk.drbd2078.out‚0 disk.drbd1673.out‚0 disk.drbd1938.out‚0! disk.dm-0.out‚657.22314453125 disk.sda1.out‚0$ disk.nvme0n1.out‚657.22314453125 cpu‚13.773064461698013* disk./.usagePercent‚32.792358561107605 disk.drbd2078.in‚0 disk.drbd2118.in‚0 disk.drbd1828.in‚0 disk.drbd1285.out‚0 disk.sda1.in‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚9.59375 disk.sda.out‚ 100.16015625 disk.drbd1965.in‚0 disk.drbd1828.out‚0 disk.drbd1584.out‚0 disk.drbd1965.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2111.in‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0% memory_percent‚41.238218545913696* proc.memory.availableMB‚38510.12109375$ disk./.usageGB‚586.3606758117676 disk.drbd1931.out‚0 disk.rbd0.in‚0 disk.drbd1226.in‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.rbd0.out‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1673.in‚0 disk.sda2.in‚ 0.68359375 proc.cpu.threads‚29 disk.dm-0.in‚9.59375 disk.drbd1226.out‚0 network.sent‚ 6.305377e+06 disk.sda.in‚ 0.68359375 disk.drbd1550.in‚0 disk.drbd1356.out‚0 disk.drbd2041.in‚0* proc.memory.percent‚17.172330617904663 network.recv‚441788 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.drbd1584.in‚0 disk.drbd1289.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 11.7216796875 disk.drbd1356.in‚0bÉ #2ù- _wandbruntime‚2887  _runtime‚2887Õƒ)Éúà ' lr-AdamW/pg1‚0.00017651370523786146 ' lr-AdamW/pg2‚0.00017651370523786146 trainer/global_step‚6849 % _timestamp‚1.7858131419228754e+09 _runtime‚2887.112820972 _step‚544i¼©ŒÎûÈ ' lr-AdamW/pg1‚0.00017651370523786146 ' lr-AdamW/pg2‚0.00017651370523786146 trainer/global_step‚6849 % _timestamp‚1.7858131419228754e+09 _runtime‚2887.112820972 _step‚544 þP¡ü2ü- _wandbruntime‚2887  _runtime‚2887ûW¸Ò'ýÌO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6027154334720 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124064819853312 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚72332923 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚114674725069472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46587944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19617333 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚120687041446488 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚123623919150592 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6027066905088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚123624023619584 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45509127 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46588180 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚117596869284864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚72342579 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _runtime‚2887.117631311 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚234 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚123623919150592 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6190220534784 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6012402180140 _step‚545 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚72342579 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚117874599318528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚120687144288972 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚236 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856ïðy×'þÑO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45509127 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46588180 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚117596869284864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚72342579 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _runtime‚2887.117631311 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚234 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚123623919150592 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6190220534784 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6012402180140 _step‚545 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚72342579 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚117874599318528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚120687144288972 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚236 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6027154334720 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124064819853312 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚72332923 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚114674725069472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46587944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19617333 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚120687041446488 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚123623919150592 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6027066905088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚123624023619584 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0¡Š'8¶2ÿ- _wandbruntime‚2887  _runtime‚2887ö+€ÕM 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚120704659092030 trainer/global_step‚6849 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45515772 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚261 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26974782 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚117614019008512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6027946524672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _runtime‚ 2887.46469396 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚117614036047872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19620211 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚124082769803264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6028033961472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117868888 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚72343446 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6028033961472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46594734 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6191121035776 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7858131422744262e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚123641965533184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚114691482281152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6027946524672 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6190988271616 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚E%~ìÕ130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19619952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117859216 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117868888 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚124082927471104 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚120704761934626 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚72353116 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117859216 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚546 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45515770 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚72343446 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚117614036047872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26974784 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚117891806435328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46594995 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚123642070009344 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45515772 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6190988271616 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19619952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117859216 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117868888 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚124082927471104 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚120704761934626 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚72353116 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117859216 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚546 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45515770 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚72343446 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚117614036047872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26974784 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚117891806435328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46594995 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚123642070009344 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45515772 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45515772 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚261 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26974782 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚117614019008512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6027946524672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _runtime‚ 2887.46469396 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚117614036047872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19620211 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚124082769803264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6028033961472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117868888 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚72343446 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6028033961472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46594734 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6191121035776 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7858131422744262e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚123641965533184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚114691482281152¢ÿ{ó2‚- _wandbruntime‚2887  _runtime‚2887Zk<ƒü 0loss/SRqqt1/detection_loss‚1.075487541412381 trainer/global_step‚6849 /loss/FRt2/assignment_loss‚2.982529536634684 1loss/FBt2/assignment_loss‚0.02457479497816814 1loss/SRqqt2/detection_loss‚2.5173057954320135 /loss/FBt1/detection_loss‚1.3902025480727187 _step‚547 0loss/FRt1/assignment_loss‚3.2920078487957225 2loss/SRqqt2/assignment_loss‚1.3939306328401846 %loss/total_loss‚20.25940187679797 _runtime‚2887.465430214 0loss/FBt1/assignment_loss‚0.3974170220027291 /loss/FRt1/detection_loss‚2.3823406850590425 /loss/FBt2/detection_loss‚1.0726886992884237 epoch‚0 % _timestamp‚1.7858131422756324e+09 1loss/SRqqt1/assignment_loss‚1.132291848456638 .loss/FRt2/detection_loss‚2.598624923825264îQè„ %loss/total_loss‚20.25940187679797 _runtime‚2887.465430214 0loss/FBt1/assignment_loss‚0.3974170220027291 /loss/FRt1/detection_loss‚2.3823406850590425 /loss/FBt2/detection_loss‚1.0726886992884237 epoch‚0 % _timestamp‚1.7858131422756324e+09 1loss/SRqqt1/assignment_loss‚1.132291848456638 .loss/FRt2/detection_loss‚2.598624923825264 0loss/SRqqt1/detection_loss‚1.075487541412381 trainer/global_step‚6849 /loss/FRt2/assignment_loss‚2.982529536634684 1loss/FBt2/assignment_loss‚0.02457479497816814 1loss/SRqqt2/detection_loss‚2.5173057954320135 /loss/FBt1/detection_loss‚1.3902025480727187 _step‚547 0loss/FRt1/assignment_loss‚3.2920078487957225 2loss/SRqqt2/assignment_loss‚1.3939306328401846£ËÚ†dœ…‚(:‘ ±ÅÓ„ÒŸƒ gpu.0.gpu‚29.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚128.563' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.85433333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0É1Èg†‚(:Ü  ±ÅÓ×æƒ disk.drbd2118.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd1931.out‚0 disk.drbd1550.out‚0 network.sent‚ 6.35063e+06 network.recv‚444065 disk.drbd1817.in‚0 disk.drbd1289.in‚0 disk.sda2.out‚ 100.49609375 disk.drbd1356.in‚0 disk.drbd1931.in‚0 disk.drbd1285.in‚0 disk.drbd1226.out‚0 disk.rbd0.out‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚9.59375 disk.drbd1289.out‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1673.in‚0 disk.sda.in‚ 0.68359375 disk.sda.out‚ 100.49609375 disk.drbd1965.in‚0 disk.drbd2078.in‚0 disk.drbd1510.out‚0* proc.memory.percent‚17.172330617904663 cpu‚13.77235608713314 disk.rbd1.out‚ 11.7216796875 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 proc.cpu.threads‚29 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.drbd1550.in‚0 disk.drbd2118.in‚0 disk.dm-0.out‚657.5185546875 disk.drbd2111.in‚0 disk.sda1.in‚0 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd1226.in‚0 disk.drbd1552.out‚0 disk.drbd1938.out‚0) proc.memory.availableMB‚ 38510.0859375$ disk./.usageGB‚586.3606567382812# disk.nvme0n1.out‚657.5185546875 disk.drbd1584.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd2041.in‚0 disk.drbd1965.out‚0 disk.drbd2078.out‚0 disk.drbd1828.in‚0 disk.dm-0.in‚9.59375( disk./.usagePercent‚32.7923574944184% memory_percent‚41.238272190093994 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0ˆë×2‡- _wandbruntime‚2907  _runtime‚29072ER¡ÉˆÃ _step‚548 ' lr-AdamW/pg1‚0.00017780231456212677 ' lr-AdamW/pg2‚0.00017780231456212677 trainer/global_step‚6899 % _timestamp‚1.7858131626285439e+09 _runtime‚2907.818496692ŸµÍ«Î‰È _step‚548 ' lr-AdamW/pg1‚0.00017780231456212677 ' lr-AdamW/pg2‚0.00017780231456212677 trainer/global_step‚6899 % _timestamp‚1.7858131626285439e+09 _runtime‚2907.818496692¤ ]©2Š- _wandbruntime‚2907  _runtime‚2907p_Ò'‹ÌO JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27167697 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚118455207435264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19761132 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6056204840524 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45841277 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚72870581 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6071063040512 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚72870581 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚72860925 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚115511750724832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚124526253436416 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 118711860 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46928827 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚121567938525996 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚72860925 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7858131626294897e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚124526357905408 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45841277 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46929087 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6056290643648 _runtime‚2907.822631563 _step‚549 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚118734930254848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 trainer/global_step‚6899 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19761390 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124970322925568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚118734955158528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6071063040512 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚124970167347712 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6235367767040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45841279 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚118455190395904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251ÙDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19761390 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124970322925568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚118734955158528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6071063040512 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚124970167347712 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6235367767040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45841279 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚118455190395904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27167697 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚118455207435264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19761132 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6056204840524 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.curres=òent‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45841277 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚72870581 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6071063040512 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚72870581 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚72860925 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚115511750724832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚124526253436416 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 118711860 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46928827 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚121567938525996 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚72860925 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7858131626294897e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚124526357905408 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658¥A³_2- _wandbruntime‚2908  _runtime‚2908>¡?+Ú&ŽÔM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚6899 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚72871448 # _timestamp‚1.78581316297921e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19764289 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚550 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45847922 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚124544299819008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚115528490897152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 118719368 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45847920 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚124544404295168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6236272314880 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚121585556171538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚72881118 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 118729040 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45847922 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚263 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6071942660096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6057168116982 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46935923 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚118752162275328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚72881118 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚121585659014134 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46935658 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6072030096896æ©ðß&ÙM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45847922 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚124544299819008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚115528490897152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 118719368 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45847920 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚124544404295168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6236272314880 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚121585556171538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚72881118 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 118729040 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45847922 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚263 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6071942660096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6057168116982 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46935923 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚118752162275328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚72881118 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚121585659014134 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46935658 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6072030096896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚6899 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚72871448 # _timestamp‚1.78581316297921e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19764289 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚550 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚119313528546304 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚125428692191232 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 119572012 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚119313545585664 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27364547 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47268305 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚125428692191232 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47268579 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 % _timestamp‚1.7858131833285098e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚119313545585664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46173429 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚73398583 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚122448835605504 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚125875859681280 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6100093304032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119562354 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚119595286094848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46173427 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6280418008576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6115146605568 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚73388927 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27364545 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 _step‚553 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6115059175936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚125428587722240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6115059175936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚73388927 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19903760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119595310998528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚122448938447988 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291í§×'œÑO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46173429 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46173427 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚274 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚116348742301472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6115146605568 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19904032 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6280548682752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚125428587722240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚119313528546304 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6100179107156 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚119313528546304 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚125428692191232 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 119572012 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚119313545585664 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27364547 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47268305 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚125428692191232 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47268579 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 % _timestamp‚1.7858131833285098e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚119313545585664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46173429 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚73398583 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚122448835605504 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚125875859681280 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6100093304032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119562354 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚119595286094848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46173427 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6280418008576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6115146605568 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚73388927 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27364545 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 _step‚553 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6115059175936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚125428587722240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6115059175936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚73388927 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19903760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119595310998528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚122448938447988 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0©à Z2- _wandbruntime‚2928  _runtime‚2928j°G«Ü&žÖM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚125446634104832 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119579520 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚6949 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 119589192 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚125446738580992 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚125893969281536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚119330695309312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27368484 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚119612493211648 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47275374 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚116365482473792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6115938795520 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚554 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚249 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚73409120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46180072 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚119612518115328 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚73399450 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47275125 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚125893815808000 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27368482 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚116365499513152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚119330695309312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚119330712348672 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119589192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6281322596352 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6100970777254 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚73409120 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚122466556093642 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚73409120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46180072 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚119612518115328 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚73399450 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47275125 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚125893815808000 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27368482 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚116365499513152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.fuWê|Ireed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚119330695309312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚119330712348672 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119589192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6281322596352 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6100970777254 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚73409120 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚122466556093642 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚125446634104832 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119579520 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 trainer/global_step‚6949 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 119589192 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚125446738580992 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚125893969281536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚119330695309312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27368484 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚119612493211648 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47275374 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚116365482473792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6115938795520 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚554ª‡š§2 - _wandbruntime‚2928  _runtime‚2928n3¿º¡þ &loss/total_loss‚21.083869702346064 _step‚555 2loss/SRqqt2/assignment_loss‚1.2721917815506458 3loss/FBt2/assignment_loss‚0.0013996360376787715 .loss/FBt2/detection_loss‚0.981638449068005 trainer/global_step‚6949 % _timestamp‚1.7858131836813922e+09 0loss/FRt1/assignment_loss‚2.7630888978137245 0loss/FBt1/assignment_loss‚0.1706452373565688 0loss/SRqqt2/detection_loss‚2.171499563846737 epoch‚0 /loss/FRt2/assignment_loss‚3.152977565239216 .loss/FRt1/detection_loss‚2.537325697043274 /loss/FRt2/detection_loss‚2.4275597265396995 0loss/SRqqt1/detection_loss‚1.168748403739208 /loss/FBt1/detection_loss‚3.1832394429615567 _runtime‚2928.871181576 2loss/SRqqt1/assignment_loss‚1.2535553011497496dˆÂ ¢ƒ 0loss/FRt1/assignment_loss‚2.7630888978137245 0loss/FBt1/assignment_loss‚0.1706452373565688 0loss/SRqqt2/detection_loss‚2.171499563846737 epoch‚0 /loss/FRt2/assignment_loss‚3.152977565239216 .loss/FRt1/detection_loss‚2.537325697043274 /loss/FRt2/detection_loss‚2.4275597265396995 0loss/SRqqt1/detection_loss‚1.168748403739208 /loss/FBt1/detection_loss‚3.1832394429615567 _runtime‚2928.871181576 2loss/SRqqt1/assignment_loss‚1.2535553011497496 &loss/total_loss‚21.083869702346064 _step‚555 2loss/SRqqt2/assignment_loss‚1.2721917815506458 3loss/FBt2/assignment_loss‚0.0013996360376787715 .loss/FBt2/detection_loss‚0.981638449068005 trainer/global_step‚6949 % _timestamp‚1.7858131836813922e+09«)Y‘e£‚(:Ú  ʱÅÓãï·ƒ disk.drbd1285.in‚0 disk.drbd2041.in‚0 disk.sda.in‚ 0.68359375 disk.sda.out‚ 101.48828125 disk.drbd1965.in‚0 disk.drbd1817.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 11.91015625# memory_percent‚41.2392258644104! disk.dm-0.out‚658.22314453125$ proc.memory.rssMB‚11254.05859375 disk.drbd1356.in‚0 disk.drbd1550.in‚0 disk.drbd1289.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd2041.out‚0 disk.drbd1226.in‚0$ disk.nvme0n1.out‚658.22314453125) proc.memory.availableMB‚ 38509.4609375( disk./.usagePercent‚32.7923574944184$ disk./.usageGB‚586.3606567382812 disk.drbd1584.in‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 network.sent‚ 6.44383e+06 disk.drbd1552.in‚0 disk.drbd1938.in‚0 disk.drbd2078.in‚0 disk.sda1.in‚0 disk.drbd1828.in‚0 disk.nvme0n1.in‚9.59375* proc.memory.percent‚17.172330617904663 disk.drbd1356.out‚0 disk.drbd1931.in‚0 disk.drbd1510.out‚0 disk.drbd2118.out‚0 disk.sda2.out‚ 101.48828125 proc.cpu.threads‚29 disk.drbd1938.out‚0 network.recv‚449796 disk.drbd1584.out‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.dm-0.in‚9.59375 disk.drbd1552.out‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd1673.in‚0 disk.sda1.out‚0 disk.drbd1828.out‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 cpu‚13.77069259154527Í网¤‚(:† ʱÅÓÝ´ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚124.911' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚41.637 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‹ª p2¥- _wandbruntime‚2949  _runtime‚2949±{øǦÁ & lr-AdamW/pg1‚0.0001803795332106574 & lr-AdamW/pg2‚0.0001803795332106574 trainer/global_step‚6999 % _timestamp‚1.7858132042717557e+09 _runtime‚2949.461696828 _step‚556(“¤ÈÌ§Æ & lr-AdamW/pg2‚0.0001803795332106574 trainer/global_step‚6999 % _timestamp‚1.7858132042717557e+09 _runtime‚2949.461696828 _step‚556 & lr-AdamW/pg1‚0.0001803795332106574¬ ^G2¨- _wandbruntime‚2949  _runtime‚2949Ó+ÉéÒ'©ÌO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6325643817984 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚6999 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 120422506 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚126330922008064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚120171866696704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46505579 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚126331026477056 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚73916929‘ƒ¥Ì×'ªÑO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚126781432942080 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46505579 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47608430 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚557 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚120171883736064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6159055311360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚120171866696704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚126330922008064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚120171866696704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46505579 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚126331026477056 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚73916929 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6325643817984 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚6999 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 120422506 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47614967 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6160022367744 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20049635 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚123347350330554 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120439672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚2949.812750853 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6159934930944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚126799408107008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120449344 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20049892 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚73937122 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚120472849051648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6144859240762 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚6999 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚117202491089792 % _timestamp‚1.7858132046224651e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46512222 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚117202508129152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46512222 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6326689722368 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚73937122 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120449344 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46512220 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚73927452 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27565334 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚123347453173150 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚126799563677696 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27565332 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚558 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚120189033459712 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6160022367744 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚120472873955328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 120439672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6326559055360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6159934930944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47615226 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚120189050499072 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚120189033459712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6144945043998 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚120189050499072 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚126349072866816Úò \á&­ÛM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47614967 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6160022367744 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20049635 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚123347350330554 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120439672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚2949.812750853 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6159934930944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚126799408107008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120449344 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20049892 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚73937122 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚120472849051648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6144859240762 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚6999 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚117202491089792 % _timestamp‚1.7858132046224651e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46512222 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚117202508129152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46512222 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6326689722368 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚73937122 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120449344 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46512220 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚73927452 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27565334 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚123347453173150 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚126799563677696 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27565332 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚558 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚120189033459712 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6160022367744 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚120472873955328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 120439672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6326559055360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6159934930944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47615226 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚120189050499072 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚120189033459712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6144945043998 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚120189050499072 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚126349072866816 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46512220 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46837729 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚74454587 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47948928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚121316022678528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46837727 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚127233256293888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚127686987429376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _runtime‚2970.015622539 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 121282658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚124210629764520 _step‚561 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚127233360762880 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚121030221886464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚121315997774848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStattÒá sMonitor.on_train_batch_start/allocation.large_pool.allocated‚46837729 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚74444931 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚121030221886464 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6370964750848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46837729 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚74454587 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47948928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚121316022678528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46837727 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚127233256293888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚127686987429376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _runtime‚2970.015622539 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 121282658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚124210629764520 _step‚561 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚127233360762880 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚121030221886464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚121315997774848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46837729 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚74444931 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚121030221886464 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6370964750848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚127251407152640 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6371727826432 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚118039499705792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚74465124 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 trainer/global_step‚7049 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46844372 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47955854 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27762184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚121047371610112 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚127251407152640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚121047388649472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20193670 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6188833507506 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6204018503168 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121309496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6204018503168 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚127705094580224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚121333229795328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121309496 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46844370 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚121047371610112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 121299824 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46844370 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47955602 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6188747704270 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚74455454 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚127251302676480 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚127704932718080 _step‚562 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6371864784896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858132251756642e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20193420 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 121299824 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6203931066368 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚74465124 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0˜ß‡Ï%½ÛM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6203931066368 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚74465124 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚2970.365935401 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚127251407152640 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6371727826432 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚118039499705792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚74465124 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 trainer/global_step‚7049 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46844372 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47955854 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27762184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚121047371610112 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚127251407152640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚121047388649472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20193670 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6188833507506 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6204018503168 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121309496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6204018503168 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚127705094580224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚121333229795328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121309496 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46844370 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚121047371610112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 121299824 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46844370 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47955602 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6188747704270 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚74455454 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚127251302676480 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚127704932718080 _step‚562 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6371864784896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 % _timestamp‚1.7858132251756642e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20193420 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 121299824²ìè™Ò2¾- _wandbruntime‚2970  _runtime‚2970e”¿ÿ _step‚563 /loss/SRqqt2/assignment_loss‚1.1778942912817 3loss/FBt2/assignment_loss‚0.0009613430234338693 /loss/FRt1/detection_loss‚2.5097670021313654 /loss/FBt2/detection_loss‚0.6441897108383915 epoch‚0 _runtime‚2970.366671115 /loss/FRt2/assignment_loss‚3.431874900272018 1loss/SRqqt1/detection_loss‚0.9647937297480169 1loss/SRqqt2/detection_loss‚2.5520978038509687 /loss/FBt1/detection_loss‚1.8487832662649453 0loss/FRt1/assignment_loss‚3.0655002065851717 /loss/FRt2/detection_loss‚2.7005791883719596 trainer/global_step‚7049 2loss/SRqqt1/assignment_loss‚1.1003245154846144 1loss/FBt1/assignment_loss‚0.01035797514305159 &loss/total_loss‚20.007123932995636 $ _timestamp‚1.785813225176885e+09ëÄ®4 À„ /loss/FBt1/detection_loss‚1.8487832662649453 0loss/FRt1/assignment_loss‚3.0655002065851717 /loss/FRt2/detection_loss‚2.7005791883719596 trainer/global_step‚7049 2loss/SRqqt1/assignment_loss‚1.1003245154846144 1loss/FBt1/assignment_loss‚0.01035797514305159 &loss/total_loss‚20.007123932995636 $ _timestamp‚1.785813225176885e+09 _step‚563 /loss/SRqqt2/assignment_loss‚1.1778942912817 3loss/FBt2/assignment_loss‚0.0009613430234338693 /loss/FRt1/detection_loss‚2.5097670021313654 /loss/FBt2/detection_loss‚0.6441897108383915 epoch‚0 _runtime‚2970.366671115 /loss/FRt2/assignment_loss‚3.431874900272018 1loss/SRqqt1/detection_loss‚0.9647937297480169 1loss/SRqqt2/detection_loss‚2.5520978038509687³œ¡çfœÁ‚(:‘ ÷±ÅÓæ°«ƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.027' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.34233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ho˜k‚(:à  ÷±ÅÓÀØ·ƒ disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.rbd1.out‚ 12.0361328125 disk.nvme0n1.in‚9.59375 disk.drbd1931.in‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1356.in‚0 disk.drbd2111.out‚0$ memory_percent‚41.23801589012146 disk.drbd1584.in‚0# disk.nvme0n1.out‚658.9482421875 disk.drbd1226.out‚0 disk.drbd1828.out‚0 disk.drbd2118.in‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd1673.in‚0 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.drbd1550.out‚0 disk.drbd1828.in‚0 disk.drbd1289.out‚0$ proc.memory.rssMB‚11254.05859375$ disk./.usageGB‚586.3606605529785 network.recv‚456728 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd1938.out‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0* proc.memory.percent‚17.172330617904663 cpu‚13.769197835210125 disk.sda.in‚ 0.68359375 disk.dm-0.in‚9.59375 disk.dm-0.out‚658.9482421875 disk.drbd1552.out‚0 disk.drbd1817.in‚0* proc.memory.availableMB‚38510.25390625 proc.cpu.threads‚29 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0 disk.sda2.out‚ 102.38671875 network.sent‚ 6.568352e+06* disk./.usagePercent‚32.792357707756246 disk.drbd2078.in‚0 disk.drbd1510.in‚0 disk.drbd1552.in‚0 disk.sda.out‚ 102.38671875 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.sda1.in‚0¡$Dœ2Ã- _wandbruntime‚2990  _runtime‚2990â„ÔÏÅÄ¿ _runtime‚2990.681099027 _step‚564 % lr-AdamW/pg1‚0.000182956751859188 % lr-AdamW/pg2‚0.000182956751859188 trainer/global_step‚7099 % _timestamp‚1.7858132454911675e+09è¸)7ÊÅÄ % _timestamp‚1.7858132454911675e+09 _runtime‚2990.681099027 _step‚564 % lr-AdamW/pg1‚0.000182956751859188 % lr-AdamW/pg2‚0.000182956751859188 trainer/global_step‚7099´=½ïC2Æ- _wandbruntime‚2990  _runtime‚2990Ï1!ÖÒ'ÇÌO WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 122142810 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚128592557553152 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚565 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚74982589 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27955097 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚125091629686512 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚121888542997504 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚118859785188832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6247047582208 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6247135011840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚128135695048704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚74972933 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27955095 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚128135590579712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚121888560036864 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚128592408266752 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47169877 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚122176353614848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚125091526844028 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47169879 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚74982589 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚118859768149472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20333417 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚7099 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚128135695048704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚128135590579712 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6247047582208 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20333672 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚122176378518528ß©ÊO×'ÈÑO OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6416179034624 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 124382720 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6231844497680 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚74972933 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 122152468 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48288512 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47169879 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858132454920635e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚74972933 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27955095 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚128135590579712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚121888560036864 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚128592408266752 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47169877 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚122176353614848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚125091526844028 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47169879 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚74982589 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚118859768149472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20333417 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚7099 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚128135695048704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚128135590579712 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6247047582208 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20333672 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚122176378518528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 122142810 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚128592557553152 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _step‚565 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚74982589 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27955097 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚125091629686512 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚121888542997504 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚118859785188832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6247047582208 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6247135011840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚128135695048704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚263 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚121905709760512 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20336538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚7099 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚74993126 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚128153741438464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 $ _timestamp‚1.785813245841908e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚128153741438464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚2991.032176318 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 122169648 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚128153636962304 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚74993126 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20336277 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚122193585635328 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚121905709760512 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚125109144489570 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48295309 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 1044©Þ³½ 76160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27959032 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6248014638592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚122193560731648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚118876508321792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122169648 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27959034 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6417090234368 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47176520 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0Öß}wà&ËÚM WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6248014638592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚122193560731648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚118876508321792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122169648 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27959034 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6417090234368 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47176520 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚263 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚121905709760512 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20336538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚7099 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚74993126 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚128153741438464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 $ _timestamp‚1.785813245841908e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚128153741438464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚2991.032176318 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 122169648 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚128153636962304 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚74993126 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20336277 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚122193585635328 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚121905709760512 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚125109144489570 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48295309 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27959032¶”ƒ©52Ì- _wandbruntime‚2991  _runtime‚2991;?WÍ‚ _step‚567 0loss/FRt2/assignment_loss‚3.3972778576676563 1loss/SRqqt1/detection_loss‚1.0788556651106842 1loss/SRqqt2/detection_loss‚2.0069231917632036 0loss/FRt1/assignment_loss‚2.9760028399638276 2loss/SRqqt2/assignment_loss‚1.3486599598465294 2loss/FBt1/assignment_loss‚0.002561457358979169 2loss/FBt2/assignment_loss‚0.006350191411514875 .loss/FRt2/detection_loss‚2.550874097372896 epoch‚0 trainer/global_step‚7099 _runtime‚2991.032894045 2loss/SRqqt1/assignment_loss‚1.2289863643090024 /loss/FRt1/detection_loss‚2.7373803947398914 /loss/FBt1/detection_loss‚1.1731926649808884 $ _timestamp‚1.785813245843108e+09 /loss/FBt2/detection_loss‚1.2545090107263863 &loss/total_loss‚19.761573695251457 6> · /loss/FBt1/detection_loss‚1.1731926649808884 $ _timestamp‚1.785813245843108e+09 /loss/FBt2/detection_loss‚1.2545090107263863 &loss/total_loss‚19.761573695251457 _step‚567 0loss/FRt2/assignment_loss‚3.3972778576676563 1loss/SRqqt1/detection_loss‚1.0788556651106842 1loss/SRqqt2/detection_loss‚2.0069231917632036 0loss/FRt1/assignment_loss‚2.9760028399638276 2loss/SRqqt2/assignment_loss‚1.3486599598465294 2loss/FBt1/assignment_loss‚0.002561457358979169 2loss/FBt2/assignment_loss‚0.006350191411514875 .loss/FRt2/detection_loss‚2.550874097372896 epoch‚0 trainer/global_step‚7099 _runtime‚2991.032894045 2loss/SRqqt1/assignment_loss‚1.2289863643090024 /loss/FRt1/detection_loss‚2.7373803947398914·i¤æœÏ‚(:‘ †²Åӂムgpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚123.832' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.27733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0*…û¡eЂ(:Ú  †²ÅÓ Ì؃ disk.nvme0n1.in‚9.59375 disk.drbd1510.out‚0' proc.memory.availableMB‚ 38510.21875 disk.drbd1226.in‚0# disk.nvme0n1.out‚659.3310546875 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd2111.in‚0 disk.drbd2118.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1584.in‚0 disk.sda.in‚ 0.68359375 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1817.out‚0 disk.drbd1285.out‚0 disk.dm-0.out‚659.3310546875 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.drbd1510.in‚0 network.recv‚458996* disk./.usagePercent‚32.792360907823856 disk.rbd1.out‚ 12.1845703125 disk.drbd1931.out‚0 disk.sda.out‚ 102.7421875 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd1285.in‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.drbd1828.in‚0 disk.drbd1289.out‚0 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.dm-0.in‚9.59375 disk.sda1.in‚0$ memory_percent‚41.23806953430176 disk.sda2.out‚ 102.7421875 disk.drbd1828.out‚0 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.drbd1673.in‚0 cpu‚13.76877492379753 disk.rbd0.out‚0 proc.cpu.threads‚29 network.sent‚ 6.603057e+06 disk.drbd1356.in‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1817.in‚0$ disk./.usageGB‚586.3607177734375 disk.drbd2111.out‚0 disk.drbd1552.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1289.in‚0 disk.rbd1.in‚0ëܼ2Ñ- _wandbruntime‚3011  _runtime‚3011(}BÇÒÁ & lr-AdamW/pg2‚0.0001842453611834533 trainer/global_step‚7149 % _timestamp‚1.7858132661946743e+09 _runtime‚3011.384621201 _step‚568 & lr-AdamW/pg1‚0.0001842453611834533Õâ.ÌÓÆ trainer/global_step‚7149 % _timestamp‚1.7858132661946743e+09 _runtime‚3011.384621201 _step‚568 & lr-AdamW/pg1‚0.0001842453611834533 & lr-AdamW/pg2‚0.0001842453611834533¸wE¯c2Ô- _wandbruntime‚3011  _runtime‚3011Iu)ÚÒ'ÕÌO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚75500935 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _step‚569 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6291043717632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48629516 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚75510591 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129038029334528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚75500935 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚261 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚129497970943488 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6291131147264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123002962 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚75510591 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6275732961188 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28151945 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28151947 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47502029 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 % _timestamp‚1.7858132661955814e+09 _runtime‚3011.389636752 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20477310 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚123036734358528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚122746881147904 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6291043717632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚122746881147904 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48629255 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129037924865536 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚122746898187264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6461394259968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0õ¹ü —ÖÑO QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚259 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚125972423923536 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 123012620 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6461261488640 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚119696776765472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚129038029334528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 trainer/global_step‚7149 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6291131147264 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6275647158064 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚122746898187264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚123036709454848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚122746881147904 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6291043717632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚122746881147904 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48629255 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129037924865536 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚122746898187264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6461394259968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚75500935 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _step‚569 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6291043717632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48629516 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚75510591 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129038029334528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚75500935 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚261 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚129497970943488 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_byteDäŠ@s.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6291131147264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123002962 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚75510591 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6275732961188 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28151945 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28151947 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47502029 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 % _timestamp‚1.7858132661955814e+09 _runtime‚3011.389636752 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20477310 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚123036734358528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123029800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 trainer/global_step‚7149 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28155884 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858132665446784e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123020128 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚122764047910912 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123029800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28155882 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚119713516937792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚129056075724288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚122764064950272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6292010774016 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6276610434522 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48636047 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20480410 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129056075724288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47508670 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚75521128 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6291923337216 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚122764064950272 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20480165 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚129516079051776 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47508672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48636294 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47508672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6462162480128 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚129055971248128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _runtime‚3011.735005394 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚122764047910912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚75511458 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6291923337216 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚75511458 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚129516236719616 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚125990041569078 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚123053941475328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6291923337216 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚75511458 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚129516236719616 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚125990041569078 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚123053941475328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123029800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 trainer/global_step‚7149 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28155884 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858132665446784e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123020128 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚122764047910912 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123029800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28155882 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚119713516937792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚247 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚129056075724288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚122764064950272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6292010774016 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6276610434522 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48636047 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20480410 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129056075724288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47508670 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚75521128 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6291923337216 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚122764064950272 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20480165 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚129516079051776 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47508672ºÆzW2Ú- _wandbruntime‚3011  _runtime‚3011—Z®šÛþ trainer/global_step‚7149 2loss/FBt1/assignment_loss‚0.004099396094261465 3loss/FBt2/assignment_loss‚0.0007559408116551677 % _timestamp‚1.7858132665459163e+09 /loss/FRt1/assignment_loss‚2.612465989022028 .loss/FRt1/detection_loss‚2.487434143289214 /loss/SRqqt2/detection_loss‚2.01967024912729 /loss/FBt2/detection_loss‚0.9188273800673736 %loss/total_loss‚20.66748284528282 _step‚571 2loss/SRqqt1/assignment_loss‚1.1201723247166184 2loss/SRqqt2/assignment_loss‚1.1885707094607985 1loss/SRqqt1/detection_loss‚1.0650101694760912 /loss/FBt1/detection_loss‚3.6360538601875305 epoch‚0 _runtime‚3011.735704903 /loss/FRt2/assignment_loss‚2.915211909437833 .loss/FRt2/detection_loss‚2.699210773592126§cK ܃ /loss/FRt1/assignment_loss‚2.612465989022028 .loss/FRt1/detection_loss‚2.487434143289214 /loss/SRqqt2/detection_loss‚2.01967024912729 /loss/FBt2/detection_loss‚0.9188273800673736 %loss/total_loss‚20.66748284528282 _step‚571 2loss/SRqqt1/assignment_loss‚1.1201723247166184 2loss/SRqqt2/assignment_loss‚1.1885707094607985 1loss/SRqqt1/detection_loss‚1.0650101694760912 /loss/FBt1/detection_loss‚3.6360538601875305 epoch‚0 _runtime‚3011.735704903 /loss/FRt2/assignment_loss‚2.915211909437833 .loss/FRt2/detection_loss‚2.699210773592126 trainer/global_step‚7149 2loss/FBt1/assignment_loss‚0.004099396094261465 3loss/FBt2/assignment_loss‚0.0007559408116551677 % _timestamp‚1.7858132665459163e+09»áÄ^‘Ý‚(:† •²ÅÓÆÏ“ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚103.299' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚34.433 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿ð½æhÞ‚(:Ý  •²ÅÓͶك disk.drbd1673.in‚0$ disk./.usageGB‚586.3607215881348 disk.drbd2118.in‚0 disk.drbd1550.out‚0* proc.memory.availableMB‚38509.93359375* proc.memory.percent‚17.172330617904663 disk.drbd1938.in‚0 disk.drbd1356.in‚0 disk.sda1.in‚0 disk.drbd1673.out‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 proc.cpu.threads‚29 network.recv‚461437 disk.drbd1817.out‚0 disk.rbd1.out‚ 12.1845703125 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1552.in‚0 disk.drbd1817.in‚0 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0! disk.dm-0.out‚660.12841796875( disk./.usagePercent‚32.7923611211617 disk.sda2.out‚ 103.2890625 disk.rbd1.in‚0 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.drbd1510.in‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚9.59375 disk.drbd1584.out‚0 disk.rbd0.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2078.in‚0 network.sent‚ 6.648098e+06 disk.sda2.in‚ 0.68359375$ disk.nvme0n1.out‚660.12841796875 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.sda.out‚ 103.2890625 disk.drbd1550.in‚0 disk.dm-0.in‚9.59375 disk.drbd1510.out‚0 disk.drbd1584.in‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0$ memory_percent‚41.23850464820862 disk.drbd2111.out‚0 cpu‚13.76815543764316 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1289.in‚0 disk.drbd1938.out‚0 disk.drbd1828.in‚0xú·Œß‚(:’ ¤²ÅÓ¼‰Šƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚112.193' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.397666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ï~¨bà‚(:×  ¤²ÅÓńσ disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1584.in‚0 disk.drbd2078.out‚0 disk.dm-0.out‚ 660.43359375 disk.drbd1817.out‚0 cpu‚13.76770444021476! disk.nvme0n1.out‚ 660.43359375 disk.sda.in‚ 0.68359375 disk.sda.out‚ 103.8671875 disk.drbd1965.out‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0$ memory_percent‚41.23850464820862) disk./.usagePercent‚32.79235898778329 disk.drbd1356.in‚0 disk.drbd1584.out‚0 disk.drbd1550.in‚0 disk.drbd1510.in‚0 disk.drbd1552.in‚0 disk.rbd0.out‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd2041.in‚0 disk.drbd2078.in‚0 disk.dm-0.in‚9.59375 disk.drbd1289.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1226.out‚0 disk.drbd2111.out‚0 disk.rbd1.out‚ 12.216796875$ proc.memory.rssMB‚11254.05859375 disk.drbd1285.in‚0 disk.drbd1673.in‚0* proc.memory.availableMB‚38509.93359375 disk.drbd1931.in‚0 disk.drbd1817.in‚0 disk.drbd1673.out‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.drbd2118.in‚0 disk.sda2.out‚ 103.8671875 proc.cpu.threads‚29 disk.drbd1828.in‚0 disk.sda1.out‚0 disk.drbd1931.out‚0 disk.drbd1965.in‚0 disk.rbd1.in‚0 disk.sda2.in‚ 0.68359375 network.sent‚ 6.693127e+06 disk.drbd1356.out‚0 disk.rbd0.in‚0 disk.drbd2111.in‚0 network.recv‚463660$ disk./.usageGB‚586.3606834411621 disk.nvme0n1.in‚9.59375 disk.drbd1289.out‚0 disk.drbd1550.out‚0¢Áç.2á- _wandbruntime‚3032  _runtime‚3032>Ù¾ÆâÀ & lr-AdamW/pg2‚0.0001855339705077186 trainer/global_step‚7199 $ _timestamp‚1.785813287186889e+09 _runtime‚3032.376827656 _step‚572 & lr-AdamW/pg1‚0.0001855339705077186ÊPbËãÅ $ _timestamp‚1.785813287186889e+09 _runtime‚3032.376827656 _step‚572 & lr-AdamW/pg1‚0.0001855339705077186 & lr-AdamW/pg2‚0.0001855339705077186 trainer/global_step‚7199¼>XLñ2ä- _wandbruntime‚3032  _runtime‚3032s)õ5 åËO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚123605236337664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚76028937 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 3032.38163274 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129940363620352 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48969465 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚76028937 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚238 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚123605219298304 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129940259151360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚130403659000320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚120533802420832 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚130403507616768 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858132871877842e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6319535621572 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚123605236337664 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚126853321003044 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6335127282688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20620430 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚76038593 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816][xŒÖ'æÐO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6319535621572 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚123605236337664 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚126853321003044 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6335127282688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20620430 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚76038593 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚123605236337664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚76028937 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 3032.38163274 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129940363620352 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48969465 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚76028937 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚238 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚123605219298304 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129940259151360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚130403659000320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚120533802420832 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚130403507616768 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858132871877842e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658½J/Õ»2ç- _wandbruntime‚3032  _runtime‚3032ß< [Ü&èÖM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6336006909440 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚76049130 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚123914297315328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6320498898030 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6320413094794 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚123622403100672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚7199 _step‚574 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20623219 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123889952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20623479 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6507478567936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚126870938648586 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858132875346277e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123880280 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚129958410010112 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123889952 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6507339512320 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48975951 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚120550525553792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47840820 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28352734 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚120550542593152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚76049130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129958410010112 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6335919472640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6336006909440 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚76039460 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚3032.724956739 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚130421611923968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48976213 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚123914272411648 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47840822 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚123622386061312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 123880280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47840820 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚123622403100672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360²Ä¹w]éÛM REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47840822 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48975951 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚120550525553792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47840820 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28352734 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚120550542593152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚76049130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129958410010112 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6335919472640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6336006909440 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚76039460 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚3032.724956739 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚130421611923968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48976213 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚123914272411648 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47840822 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚123622386061312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 123880280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47840820 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚123622403100672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6336006909440 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚76049130 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚123914297315328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6320498898030 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6320413094794 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚123622403100672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚7199 _step‚574 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20623219 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123889952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20623479 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6507478567936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚126870938648586 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858132875346277e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123880280 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚129958410010112 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123889952 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6507339512320 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚130842593437184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚121370811036832 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚124463574488064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚249 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚124463557448704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48166327 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚76566595 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚76556939 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49308773 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6551761440256 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚577 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20763128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚131309207478784 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚130842697906176 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127734218082552 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚131309051900928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚124757446038528 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚7249 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚130842697906176 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20763128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚131309207478784 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚130842697906176 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127734218082552 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚131309051900928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚124757446038528 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚7249 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚130842697906176 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚130842593437184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚121370811036832 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚124463574488064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚249 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚124463557448704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48166327 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚76566595 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚76556939 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130ÁÀ¬T2õ- _wandbruntime‚3053  _runtime‚3053Ç+^ öÔM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6379915608064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚121387534169792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚124480724211712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚121387551209152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6380003044864 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124750104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127751835728094 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6380003044864 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚127751938570690 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 _step‚578 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚76567462 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6552529938944 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48172970 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48172972 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚24_QgÉÐ9 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚76577132 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48172972 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚130860639819776 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚124480724211712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 124740432 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚76577132 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49315906 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20766322 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20766073 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚131327317955584 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚131327158190592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚124480741251072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6364387361538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6364301558302 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 $ _timestamp‚1.785813308143076e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6552664800256 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124740432 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48172970 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124750104 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚130860744295936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚ 3053.33334463 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49315655 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28549582 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚124774628251648c-'–ß&÷ÙM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚124480724211712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 124740432 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚76577132 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49315906 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20766322 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20766073 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚131327317955584 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚131327158190592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚124480741251072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6364387361538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6364301558302 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 $ _timestamp‚1.785813308143076e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6552664800256 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124740432 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48172970 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124750104 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚130860744295936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚ 3053.33334463 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49315655 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28549582 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚124774628251648 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6379915608064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚121387534169792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚124480724211712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚121387551209152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6380003044864 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124750104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127751835728094 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6380003044864 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚127751938570690 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 _step‚578 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚76567462 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6552529938944 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48172970 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48172972 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚249 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚76577132 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48172972 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚130860639819776ÂBöà*2ø- _wandbruntime‚3053  _runtime‚3053’cÃùÿ 0loss/FRt1/assignment_loss‚2.8739716970339053 /loss/FRt1/detection_loss‚2.4877280465713363 /loss/FRt2/detection_loss‚2.5496542631240375 0loss/SRqqt1/detection_loss‚1.011367539745859 /loss/FBt1/detection_loss‚0.9646493850741535 _runtime‚3053.334170182 0loss/SRqqt1/assignment_loss‚0.96280990705903 1loss/SRqqt2/detection_loss‚2.0512410852353313 /loss/FBt2/detection_loss‚0.7809996378851966 &loss/total_loss‚18.283708556724026 epoch‚0 trainer/global_step‚7249 _step‚579 /loss/FRt2/assignment_loss‚3.484444920266612 $ _timestamp‚1.785813308144292e+09 2loss/SRqqt2/assignment_loss‚1.0580877921468503 2loss/FBt1/assignment_loss‚0.022060731962710634 2loss/FBt2/assignment_loss‚0.036693550619003666´ØÜÜ ú„ /loss/FRt2/detection_loss‚2.5496542631240375 0loss/SRqqt1/detection_loss‚1.011367539745859 /loss/FBt1/detection_loss‚0.9646493850741535 _runtime‚3053.334170182 0loss/SRqqt1/assignment_loss‚0.96280990705903 1loss/SRqqt2/detection_loss‚2.0512410852353313 /loss/FBt2/detection_loss‚0.7809996378851966 &loss/total_loss‚18.283708556724026 epoch‚0 trainer/global_step‚7249 _step‚579 /loss/FRt2/assignment_loss‚3.484444920266612 $ _timestamp‚1.785813308144292e+09 2loss/SRqqt2/assignment_loss‚1.0580877921468503 2loss/FBt1/assignment_loss‚0.022060731962710634 2loss/FBt2/assignment_loss‚0.036693550619003666 0loss/FRt1/assignment_loss‚2.8739716970339053 /loss/FRt1/detection_loss‚2.4877280465713363ÃSoCf›û‚(: ²ÅÓ즧ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚108.61' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚36.20333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0_:¿½hü‚(:Ý  ²ÅÓ®óσ disk.sda2.in‚ 0.68359375$ disk./.usageGB‚586.3567962646484 disk.drbd1673.in‚0 disk.drbd2078.in‚0 disk.dm-0.in‚9.59375 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.rbd1.out‚ 12.373046875 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.sda1.in‚0 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.sda.in‚ 0.68359375 disk.sda.out‚ 105.2265625 disk.drbd1285.out‚0 network.recv‚466977 disk.drbd1226.out‚0$ memory_percent‚41.23826622962952* proc.memory.percent‚17.172330617904663 disk.nvme0n1.in‚9.59375 disk.drbd2041.out‚0 disk.drbd1289.out‚0 network.sent‚ 6.741437e+06 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd2111.in‚0 proc.cpu.threads‚29 disk.drbd1938.out‚0 disk.drbd2041.in‚0 disk.drbd1828.in‚0* proc.memory.availableMB‚38510.08984375 disk.dm-0.out‚662.4267578125 disk.drbd1356.out‚0 disk.drbd1584.out‚0 disk.rbd0.out‚0 cpu‚13.766693786885826 disk.rbd1.in‚0# disk.nvme0n1.out‚662.4267578125 disk.drbd1356.in‚0 disk.drbd1931.in‚0 disk.drbd1965.out‚0 disk.rbd0.in‚0 disk.drbd1510.in‚0$ proc.memory.rssMB‚11254.05859375* disk./.usagePercent‚32.792141596523464 disk.drbd1817.in‚0 disk.drbd2111.out‚0 disk.drbd1817.out‚0 disk.sda2.out‚ 105.2265625 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.drbd1285.in‚0 disk.drbd2118.out‚0 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0?™\2ý- _wandbruntime‚3074  _runtime‚3074aírÅþ¿ & lr-AdamW/pg2‚0.0001881111891562492 trainer/global_step‚7299 $ _timestamp‚1.785813328918045e+09 _runtime‚ 3074.10797732 _step‚580 & lr-AdamW/pg1‚0.0001881111891562492Åp;ÊÿÄ & lr-AdamW/pg1‚0.0001881111891562492 & lr-AdamW/pg2‚0.0001881111891562492 trainer/global_step‚7299 $ _timestamp‚1.785813328918045e+09 _runtime‚ 3074.10797732 _step‚580ľ>¹b2€- _wandbruntime‚3074  _runtime‚3074XŠx›Ò'ÌO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48498477 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚125617776974848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚122207802613472 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚131744927723008 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚125321912638464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 % _timestamp‚1.7858133289189384e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚122207819652832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49649540 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚77084941 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚131745032192000 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚131745032192000 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48498477 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚125321912638464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28742497 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49649284 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6596811429376 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28742495 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚125321895599104 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6407312548588 trainer/global_step‚7299 _step‚581 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125583418 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚125321895599104 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20907043 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚77084941 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚128615218004544 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚128615115162060 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚131744927723008 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6407398351712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚77094597 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚3074.112634085 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125593076 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6423032123904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚132214588404224 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6423119553536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚77094597 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6423119553536 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20906789 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6423032123904 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚132214746079232 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚125617801878528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6596944200704 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0æÞ†¶í‚ÑO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚125321912638464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28742497 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49649284 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADevi\£qêê"ceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6596811429376 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28742495 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚125321895599104 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6407312548588 trainer/global_step‚7299 _step‚581 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125583418 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚125321895599104 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20907043 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚77084941 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚128615218004544 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚128615115162060 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚131744927723008 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6407398351712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚77094597 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚3074.112634085 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125593076 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6423032123904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚132214588404224 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6423119553536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚77094597 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6423119553536 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20906789 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6423032123904 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚132214746079232 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚125617801878528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6596944200704 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48498477 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚125617776974848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚122207802613472 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚131744927723008 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚125321912638464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 % _timestamp‚1.7858133289189384e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚122207819652832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49649540 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚77084941 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚131745032192000 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚131745032192000 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48498477 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251ÅÊI (2ƒ- _wandbruntime‚3074  _runtime‚30744ð€ŠÜ&„ÖM NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48505120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20909805 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚77105134 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49655991 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6408190021810 trainer/global_step‚7299 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚131762974105600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20909559 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6597846340096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 125600584 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚125635008995328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48505122 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚582 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48505120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6423999180288 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚128632835650198 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚125339062362112 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6408275825046 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49656239 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125610256 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚77105134 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6597713575936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚122224542785792 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚132232855335424 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚131763078581760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 % _timestamp‚1.7858133292645886e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚125339079401472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6423911743488 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48505122ÂDÀ á&…ÛM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6408275825046 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49656239 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125610256 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚77105134 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6597713575936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚122224542785792 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚132232855335424 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚131763078581760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 % _timestamp‚1.7858133292645886e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚125339079401472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6423911743488 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48505122 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48505120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20909805 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚77105134 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49655991 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6408190021810 trainer/global_step‚7299 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚131762974105600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20909559 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6597846340096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 125600584 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚125635008995328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48505122 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚582 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48505120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6423999180288 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚128632835650198 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚125339062362112 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392ÆVЋ÷2†- _wandbruntime‚3074  _runtime‚3074ef‡þ trainer/global_step‚7299 2loss/SRqqt2/assignment_loss‚1.2598767136888844 %loss/total_loss‚19.04834493210941 epoch‚0 _step‚583 /loss/FBt1/detection_loss‚1.4743469543755054 /loss/FBt2/detection_loss‚0.9640964079631625 0loss/FRt1/assignment_loss‚2.5175016085306803 4loss/FBt2/assignment_loss‚0.00026231776191895255 /loss/FRt1/detection_loss‚2.3549283385276794 1loss/SRqqt1/detection_loss‚0.9103023623862606 /loss/FRt2/assignment_loss‚3.213770659163941 % _timestamp‚1.7858133292657673e+09 _runtime‚ 3074.45554422 1loss/SRqqt1/assignment_loss‚0.950947227277207 0loss/FBt1/assignment_loss‚0.5648188789580673 .loss/FRt2/detection_loss‚2.630824721829836 0loss/SRqqt2/detection_loss‚2.206668741646267v ˆƒ 1loss/SRqqt1/assignment_loss‚0.950947227277207 0loss/FBt1/assignment_loss‚0.5648188789580673 .loss/FRt2/detection_loss‚2.630824721829836 0loss/SRqqt2/detection_loss‚2.206668741646267 trainer/global_step‚7299 2loss/SRqqt2/assignment_loss‚1.2598767136888844 %loss/total_loss‚19.04834493210941 epoch‚0 _step‚583 /loss/FBt1/detection_loss‚1.4743469543755054 /loss/FBt2/detection_loss‚0.9640964079631625 0loss/FRt1/assignment_loss‚2.5175016085306803 4loss/FBt2/assignment_loss‚0.00026231776191895255 /loss/FRt1/detection_loss‚2.3549283385276794 1loss/SRqqt1/detection_loss‚0.9103023623862606 /loss/FRt2/assignment_loss‚3.213770659163941 % _timestamp‚1.7858133292657673e+09 _runtime‚ 3074.45554422ÇÀÊ8(:’ ѲÅÓ𿡃 gpu.0.gpu‚41.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.442' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.480666666666664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0!O³jŠ‚(:ß  ѲÅÓû®ìƒ$ disk./.usageGB‚586.3567962646484 disk.drbd1510.out‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1285.out‚0 disk.drbd1289.out‚0 cpu‚13.766054330385094 disk.drbd2111.out‚0$ disk.nvme0n1.out‚663.00927734375 disk.drbd1931.in‚0 disk.drbd1965.in‚0 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd2111.in‚0 disk.sda1.in‚0 disk.drbd1817.in‚0 disk.nvme0n1.in‚9.59375$ memory_percent‚41.23833179473877 disk.drbd2118.in‚0 disk.drbd1828.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.drbd1828.out‚0 proc.cpu.threads‚29 disk.drbd1965.out‚0( proc.memory.availableMB‚ 38510.046875 disk.sda2.out‚ 105.73828125 disk.drbd1584.out‚0* proc.memory.percent‚17.172330617904663! disk.dm-0.out‚663.00927734375 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.sda.out‚ 105.73828125 disk.drbd1226.in‚0 disk.drbd2078.out‚0 network.recv‚469481* disk./.usagePercent‚32.792141596523464 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd1356.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1550.in‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0$ proc.memory.rssMB‚11254.05859375 network.sent‚ 6.786475e+06 disk.dm-0.in‚9.59375 disk.drbd1673.out‚0 disk.drbd2041.out‚0 disk.drbd1226.out‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.rbd1.out‚ 12.373046875 disk.drbd1550.out‚0 disk.drbd1510.in‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0;"3x›‹‚(:á  à²ÅÓ›îÁƒ disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd2078.out‚0% memory_percent‚41.238319873809814 disk.drbd1289.ô,°Ñout‚0 disk.drbd1552.in‚0 disk.drbd1817.in‚0 disk.rbd1.out‚ 12.373046875 disk.drbd1828.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1938.out‚0 proc.cpu.threads‚29 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd2041.in‚0 disk.rbd0.in‚0 network.sent‚ 6.831518e+06* disk./.usagePercent‚32.792141596523464 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.sda2.out‚ 106.40234375$ disk./.usageGB‚586.3567962646484 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0 cpu‚13.765465568968485 disk.drbd1226.out‚0 disk.drbd1938.in‚0 disk.dm-0.in‚9.59375* proc.memory.percent‚17.172330617904663 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.drbd1289.in‚0 disk.rbd0.out‚0 disk.drbd1510.in‚0 disk.drbd1965.out‚0 disk.drbd2111.out‚0$ disk.nvme0n1.out‚663.25537109375! disk.dm-0.out‚663.25537109375 network.recv‚471667 disk.sda.out‚ 106.40234375 disk.drbd1828.out‚0) proc.memory.availableMB‚ 38510.0546875 disk.sda1.in‚0 disk.drbd1356.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1285.out‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.nvme0n1.in‚9.59375 disk.drbd1584.out‚0 disk.drbd1931.out‚0¤pÒgŒ‚(:’ à²Åәƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚133.533' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.510999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ë¹Q2- _wandbruntime‚3094  _runtime‚3094ä»LÉŽÃ ' lr-AdamW/pg2‚0.00018939979848051452 trainer/global_step‚7349 % _timestamp‚1.7858133495611029e+09 _runtime‚3094.751038302 _step‚584 ' lr-AdamW/pg1‚0.00018939979848051452»¸‰ŒÎÈ % _timestamp‚1.7858133495611029e+09 _runtime‚3094.751038302 _step‚584 ' lr-AdamW/pg1‚0.00018939979848051452 ' lr-AdamW/pg2‚0.00018939979848051452 trainer/global_step‚7349ÈýÃJ·2- _wandbruntime‚3094  _runtime‚3094¬Ê%…Ñ'‘ËO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚275 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 trainer/global_step‚7349 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6451286815220 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚129496012241568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚3094.755630421 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚132647262008832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚126180233749504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28939347 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚133120274388992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6467115688960 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6642116670464 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6467028259328 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48830627 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚126478132814848 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚126180250788864 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6467115688960 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6641975510528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚123044811229472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚77612943 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚77622599 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚77612943 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚123044828268832 $ _timestamp‚1.785813349562022e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48830629 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚48830629 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚77622599 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6467028259328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚132647366477824 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 126453228 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49988411 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚273 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _step‚585Y¢ÚªÖ'’ÐO ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6467028259328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚132647366477824 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 126453228 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49988411 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚273 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _step‚585 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚275 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 trainer/global_step‚7349 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6451286815220 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚129496012241568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚3094.755630421 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚132647262008832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚126180233749504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28939347 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚133120274388992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6467115688960 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6642116670464 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6467028259328 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48830627 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚126478132814848 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚126180250788864 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6467115688960 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6641975510528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚123044811229472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚77612943 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚77622599 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚77612943 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚123044828268832 $ _timestamp‚1.785813349562022e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48830629 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚48830629 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚77622599 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130ÉY¿³ï2“- _wandbruntime‚3095  _runtime‚3095^àòx&”ÖM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚77633136 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚77633136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚133138222520320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚132665412867584 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚129513732729706 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6452078485318 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6467995315712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 trainer/global_step‚7349 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49994966 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚126495364835328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 % _timestamp‚1.7858133499165757e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48837270 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21051684 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6452164288554 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚126197417551872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚126495339931648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚3095.106874935 _step‚586 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49995206 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6643013255680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚123061551401792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚126197400512512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚133138378091008 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126470408 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚77623466 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚132665308391424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6467995315712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126470408 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48837270 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚77623466 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28943284 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21051922 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚238 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6642882588672 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚132665412867584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28943282 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚129513629887110 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚123061568441152 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.currM†*ædent‚ 104476160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640õ¶2Gá&•ÛM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚126495364835328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 % _timestamp‚1.7858133499165757e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48837270 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21051684 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6452164288554 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚126197417551872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚126495339931648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚3095.106874935 _step‚586 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49995206 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6643013255680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚123061551401792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚126197400512512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚133138378091008 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126470408 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚77623466 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚132665308391424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6467995315712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126470408 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48837270 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚77623466 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28943284 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21051922 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚238 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6642882588672 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚132665412867584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28943282 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚129513629887110 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚123061568441152 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚77633136 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚77633136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚133138222520320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚132665412867584 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚129513732729706 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6452078485318 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6467995315712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 trainer/global_step‚7349 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49994966 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0ÊÅ&02–- _wandbruntime‚3095  _runtime‚3095S S—þ 0loss/FRt1/assignment_loss‚2.6460441926663574 2loss/SRqqt2/assignment_loss‚1.2131255147712572 3loss/FBt2/assignment_loss‚0.0017745298573296963 .loss/FRt2/detection_loss‚2.645478727725836 _runtime‚3095.107603768 _step‚587 1loss/SRqqt1/assignment_loss‚1.040953665885622 .loss/FRt1/detection_loss‚2.416815886019983 0loss/SRqqt2/detection_loss‚2.209183331685407 /loss/FBt2/detection_loss‚1.2524639736954122 &loss/total_loss‚19.595571310853053 epoch‚0 trainer/global_step‚7349 0loss/FRt2/assignment_loss‚2.9265934022573323 1loss/FBt1/assignment_loss‚0.03503989743621787 1loss/SRqqt1/detection_loss‚0.8821468409796843 .loss/FBt1/detection_loss‚2.325951347872615 $ _timestamp‚1.785813349917827e+09â­ U ˜ƒ 1loss/SRqqt1/assignment_loss‚1.040953665885622 .loss/FRt1/detection_loss‚2.416815886019983 0loss/SRqqt2/detection_loss‚2.209183331685407 /loss/FBt2/detection_loss‚1.2524639736954122 &loss/total_loss‚19.595571310853053 epoch‚0 trainer/global_step‚7349 0loss/FRt2/assignment_loss‚2.9265934022573323 1loss/FBt1/assignment_loss‚0.03503989743621787 1loss/SRqqt1/detection_loss‚0.8821468409796843 .loss/FBt1/detection_loss‚2.325951347872615 $ _timestamp‚1.785813349917827e+09 0loss/FRt1/assignment_loss‚2.6460441926663574 2loss/SRqqt2/assignment_loss‚1.2131255147712572 3loss/FBt2/assignment_loss‚0.0017745298573296963 .loss/FRt2/detection_loss‚2.645478727725836 _runtime‚3095.107603768 _step‚587Ë ¤™‚(:’ ï²ÅÓ´¶¤ƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚123.883' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚41.294333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚09 B:lš‚(:á  ï²ÅÓèÛÓƒ disk.rbd1.out‚ 12.5849609375 network.sent‚ 6.834555e+06 disk.drbd1931.in‚0 disk.sda.out‚ 106.77734375 disk.drbd1510.in‚0 disk.drbd1938.out‚0$ disk./.usageGB‚586.3564491271973 disk.drbd2111.out‚0 disk.drbd1356.out‚0 disk.drbd2041.in‚0 disk.drbd1817.in‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0) proc.memory.availableMB‚ 38509.9921875 disk.drbd1356.in‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚ 106.77734375 disk.drbd1285.out‚0 disk.drbd1289.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1226.in‚0 disk.dm-0.in‚9.59375 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0) disk./.usagePercent‚32.79212218277994 disk.drbd1931.out‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0$ disk.nvme0n1.out‚663.53173828125 proc.cpu.threads‚29 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1828.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.nvme0n1.in‚9.59375% memory_percent‚41.238415241241455 network.recv‚472590 disk.drbd1584.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1550.in‚0 disk.sda2.in‚ 0.68359375 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1817.out‚0 disk.drbd1584.out‚0 disk.rbd1.in‚0 cpu‚13.765114896398007! disk.dm-0.out‚663.53173828125 disk.drbd2111.in‚0Î_•µ2›- _wandbruntime‚3115  _runtime‚3115üýÇÇœÁ _step‚588 & lr-AdamW/pg1‚0.0001906884078047798 & lr-AdamW/pg2‚0.0001906884078047798 trainer/global_step‚7399 % _timestamp‚1.7858133703350554e+09 _runtime‚3115.524985594ÐеÌÆ _runtime‚3115.524985594 _step‚588 & lr-AdamW/pg1‚0.0001906884078047798 & lr-AdamW/pg2‚0.0001906884078047798 trainer/global_step‚7399 % _timestamp‚1.7858133703350554e+09ÌRÆ>j2ž- _wandbruntime‚3115  _runtime‚3115×èö°Ñ'ŸËO QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6511111824384 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚78150601 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29136195 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚127338488654848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6495175278728 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133549596294656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚130376909321076 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 120188416 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6687175362048 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚130377012163560 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49162777 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 % _timestamp‚1.7858133703359604e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚127038588939264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚78140945 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49162777 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 145092096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29136197 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21192769 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21192523 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚127038588939264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50328966 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50328718 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚78150601 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚127038571899904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚49162779 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚127338513558528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49162779 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6687295550464 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚123881836884832 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6511024394752 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚246 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6495089475604 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚133549700763648 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚133549700763648 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚127038571899904 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 trainer/global_step‚7399 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127313380 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6511111824384 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _runtime‚ 3115.52963883 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚134025809108992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200<ŸyP  ÐO N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 120188416 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6687175362048 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚130377012163560 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49162777 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 % _timestamp‚1.7858133703359604e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚127038588939264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚78140945 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49162777 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 145092096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29136197 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21192769 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21192523 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚127038588939264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50328966 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50328718 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚78150601 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚127038571899904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚49162779 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚127338513558528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49162779 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6687295550464 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚123881836884832 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6511024394752 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚246 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6495089475604 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚133549700763648 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚133549700763648 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚127038571899904 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 trainer/global_step‚7399 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127313380 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 65111118243§Ñ4Õ†84 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 _runtime‚ 3115.52963883 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚134025809108992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6511111824384 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚78150601 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29136195 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚127338488654848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6495175278728 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133549596294656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚130376909321076 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130Íq£2¡- _wandbruntime‚3115  _runtime‚3115×4dÜ&¢ÖM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚127055738662912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6688210348544 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6511991451136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29140132 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚133567747153408 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133567642677248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49169422 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50335553 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚134043767064576 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49169420 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚127355720675328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6496052752062 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚127055755702272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚123898560017792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚590 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 127320888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21195421 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21195681 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49169422 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚78151468 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6688071292928 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚3115.875748447 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚127055738662912 trainer/global_step‚7399 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚133567747153408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6511991451136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚130394526966618 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49169420 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚127055755702272 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858133706854403e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50335815 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127330560 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 127320888 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚78161138 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚130394629809214 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6511904014336 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6495966948826‚pD|á&£ÛM @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚127055738662912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6688210348544 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6511991451136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29140132 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚133567747153408 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133567642677248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49169422 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50335553 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚134043767064576 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49169420 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚127355720675328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6496052752062 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚127055755702272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚123898560017792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚590 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 127320888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21195421 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21195681 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49169422 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚78151468 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6688071292928 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _runtime‚3115.875748447 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚127055738662912 trainer/global_step‚7399 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚133567747153408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6511991451136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚130394526966618 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49169420 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚127055755702272 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 % _timestamp‚1.7858133706854403e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50335815 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127330560 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 127320888 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚78161138 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚130394629809214 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6511904014336 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6495966948826 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360Îí‘©|2¤- _wandbruntime‚3115  _runtime‚3115`5Ç–¥ý 2loss/SRqqt1/assignment_loss‚0.9547590700745144 /loss/FBt2/detection_loss‚0.8461292433124539 2loss/FBt1/assignment_loss‚0.022252040842431597 /loss/FRt2/detection_loss‚2.6806178532540796 .loss/FBt1/detection_loss‚2.614480331649247 epoch‚0 trainer/global_step‚7399 2loss/SRqqt2/assignment_loss‚1.2157909384986851 /loss/FBt2/assignment_loss‚0.021017415423316 ,loss/FRt1/detection_loss‚2.4753783425723 &loss/total_loss‚19.238029046766755 % _timestamp‚1.7858133706866496e+09 _step‚591 0loss/FRt1/assignment_loss‚2.8209288613549592 0loss/FRt2/assignment_loss‚2.6446219637989996 1loss/SRqqt1/detection_loss‚0.9673443042289684 1loss/SRqqt2/detection_loss‚1.9747086817567998 _runtime‚3115.876595634v¯As¦‚ 1loss/SRqqt2/detection_loss‚1.9747086817567998 _runtime‚3115.876595634 2loss/SRqqt1/assignment_loss‚0.9547590700745144 /loss/FBt2/detection_loss‚0.8461292433124539 2loss/FBt1/assignment_loss‚0.022252040842431597 /loss/FRt2/detection_loss‚2.6806178532540796 .loss/FBt1/detection_loss‚2.614480331649247 epoch‚0 trainer/global_step‚7399 2loss/SRqqt2/assignment_loss‚1.2157909384986851 /loss/FBt2/assignment_loss‚0.021017415423316 ,loss/FRt1/detection_loss‚2.4753783425723 &loss/total_loss‚19.238029046766755 % _timestamp‚1.7858133706866496e+09 _step‚591 0loss/FRt1/assignment_loss‚2.8209288613549592 0loss/FRt2/assignment_loss‚2.6446219637989996 1loss/SRqqt1/detection_loss‚0.9673443042289684ÏbÕ±p‘§‚(:† þ²ÅÓ€÷¢ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚129.639' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.213 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0brrDj¨‚(:ß  þ²ÅÓ†üçƒ network.sent‚ 6.879753e+06 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.sda.out‚ 107.1484375 disk.drbd1965.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1931.out‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0$ memory_percent‚41.23900532722473$ proc.memory.rssMB‚11254.05859375 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.nvme0n1.in‚9.59375 disk.drbd1828.out‚0 disk.drbd2111.out‚0$ disk./.usageGB‚586.3584671020508 disk.drbd1584.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd2118.in‚0 disk.rbd1.out‚ 12.5849609375 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.sda2.out‚ 107.1484375$ disk.nvme0n1.out‚663.69580078125 disk.drbd1285.in‚0 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 network.recv‚475213 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.sda.in‚ 0.68359375 disk.drbd1226.in‚0 disk.dm-0.in‚9.59375 disk.drbd1289.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd2078.out‚0 cpu‚13.764492430973155 proc.cpu.threads‚29 disk.drbd2041.in‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0! disk.dm-0.out‚663.69580078125 disk.drbd1226.out‚0 disk.rbd0.out‚0) disk./.usagePercent‚32.79223503849775 disk.drbd1584.in‚0 disk.drbd1828.in‚0 disk.drbd1285.out‚0* proc.memory.availableMB‚38509.60546875 disk.drbd1356.in‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.drbd1289.out‚0N±°Öc©‚(:Ø  ³ÅÓõÿ²ƒ disk.drbd2078.in‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd1510.out‚0 disk.sda1.in‚0$ disk./.usageGB‚586.3584251403809 disk.drbd1817.out‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 107.640625! disk.dm-0.out‚663.99169921875 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.rbd1.out‚ 12.6474609375 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.drbd1285.in‚0 network.sent‚ 6.924787e+06 network.recv‚477736 disk.drbd2111.in‚0 disk.drbd1289.in‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0' proc.memory.availableMB‚ 38509.84375 cpu‚13.763959154884107 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.sda1.out‚0# memory_percent‚41.2386417388916 proc.cpu.threads‚29( disk./.usagePercent‚32.7922326917815 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.sda.in‚ 0.68359375$ proc.memory.rssMB‚11254.05859375 disk.sda.out‚ 107.640625 disk.drbd1226.in‚0 disk.drbd1673.out‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1828.out‚0 disk.drbd2111.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd1673.in‚0 disk.nvme0n1.in‚9.59375 disk.drbd1584.out‚0 disk.rbd0.out‚0 disk.drbd1828.in‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 0.68359375$ disk.nvme0n1.out‚663.99169921875 disk.rbd1.in‚0 disk.dm-0.in‚9.59375~#>œª‚(:‘ ³ÅÓ©¹ìƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚124.519' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.50633333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡Bv'2«- _wandbruntime‚3136  _runtime‚3136Ü9¹+ɬà ' lr-AdamW/pg2‚0.00019197701712904514 trainer/global_step‚7449 % _timestamp‚1.7858133913880951e+09 _runtime‚3136.578041389 _step‚592 ' lr-AdamW/pg1‚0.00019197701712904514´Åç/Î­È ' lr-AdamW/pg1‚0.00019197701712904514 ' lr-AdamW/pg2‚0.00019197701712904514 trainer/global_step‚7449 % _timestamp‚1.7858133913880951e+09 _runtime‚3136.578041389 _step‚592ÐÛÝø2®- _wandbruntime‚3136  _runtime‚3136Éâ< é¯ÌO NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6732394322944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49494927 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6732516608512 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6555107959808 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚78668947 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚234 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21337935 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚128198869398528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128173532 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚131257909243068 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚127896910050304 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49494927 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚134931238817792 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚127896927089664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚593 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚78678603 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49494929 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚78668947 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚127896927089664 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29333045 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚78678603 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 trainer/global_step‚7449 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50670982 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚3136.582634239 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚127896910050304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚131257806400584 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128163874 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚134931386007040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚134452035049472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29333047 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6539063742236 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚134451930580480 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21337703 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚134452035049472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚134451930580480 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6555020530176àƬ×'°ÑO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚127896910050304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚131257806400584 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128163874 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚134931386007040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚134452035049472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29333047 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6539063742236 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚134451930580480 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21337703 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚134452035049472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚134451930580480 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6555020530176 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6732394322944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49494927 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6732516608512 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6555107959808 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚78668947 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚234 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21337935 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚128198869398528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128173532 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚131257909243068 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚127896910050304 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49494927 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚134931238817792 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚127896927089664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚593 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚78678603 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 147189248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49494929 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚78668947 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚127896927089664 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29333045 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚78678603 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 trainer/global_step‚7449 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50670982 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚3136.582634239у«ïÛ2±- _wandbruntime‚3136  _runtime‚3136WØÞÜ&²ÖM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚78689140 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21340505 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6733429479424 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21340771 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29336984 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚3136.933507697 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6539941215570 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6555987586560 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50677755 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚128216051611648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚131275526888722 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 128190712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚127914076813312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 % _timestamp‚1.7858133917432365e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚127914093852672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚78689140 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚268 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29336982 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚124735585673152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚78679470 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50677487 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49501570 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49501572 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128190712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚127914093852672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚128216076515328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 128181040 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49501570 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚134949505994752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128181040 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6539941215570 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6555987586560 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50677755 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚128216051611648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚131275526888722 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 128190712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚127914076813312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 % _timestamp‚1.7858133917432365e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚127914093852672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚78689140 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚268 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29336982 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚124735585673152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚78679470 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50677487 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49501570 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49501572 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128190712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚127914093852672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚128216076515328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 128181040 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49501570 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚134949505994752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128181040 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21340505 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6733429479424 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21340771 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29336984 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚3136.933507697 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚135354264866304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21483597 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚128755265240064 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚79196949 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚129059200334848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚135836817026560 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚132138806322576 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858134120859432e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚79196949 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 _runtime‚3157.279637459 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6599104095232 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚128755248200704 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚135354264866304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21483597 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚128755265240064 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚79196949 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚129059200334848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚135836817026560 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚132138806322576 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858134120859432e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚79196949 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 _runtime‚3157.279637459 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6599104095232 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚128755248200704 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚128772414963712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29533832 _runtime‚3157.633282901 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚135372415725056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚128772432003072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6599983721984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29533834 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49833722 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚79217142 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6778517920768 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚135372415725056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚7499 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49833720 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51020003 TFDeviceStatsMonitor.on_train_batch_end/iBÃn1nactive_split_bytes.all.current‚ 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6599983721984 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6583743875842 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129050864 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6599896285184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚135854925372416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚251 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚129076432355328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51020256 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 129041192 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚132156423968230 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _step‚598 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚79207472 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚125572594289152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21486171 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚132156321125634 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392©z£Èá&ÁÛM ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚125572577249792 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49833720 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51020003 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6599983721984 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6583743875842 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129050864 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6599896285184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚135854925372416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚251 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚129076432355328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51020256 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 129041192 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚132156423968230 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _step‚598 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚79207472 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚125572594289152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21486171 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚132156321125634 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚79207472 % _timestamp‚1.7858134124430006e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49833720 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚135372311248896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6778654879232 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚128772414963712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29533832 _runtime‚3157.633282901 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚135372415725056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚128772432003072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6599983721984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29533834 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49833722 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚79217142 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6778517920768 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚135372415725056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚7499 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136256599152128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚129613603390464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21626799 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚129613586351104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51353801 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6822950642688 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚79724951 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6822819968512 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 % _timestamp‚1.7858134328540282e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚129613586351104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6643012801024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚7549 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6643100230656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51353544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29726747 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50159227 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50159229 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚129919581078528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6643100230656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚79734607 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚601 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚136256703621120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50159229 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816‹f° ˆ ÌÐO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚79724951 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6822819968512 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 % _timestamp‚1.7858134328540282e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚129613586351104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6643012801024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚7549 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6643100230656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51353544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29726747 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50159227 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50159229 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚129919581078528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6643100230656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚79734607 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚601 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚136256703621120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50159229 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29726745 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6626840669252 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6626754866128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6643012801024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚79734607 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136256599152128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚129613603390464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21626799 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚129613586351104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51353801 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6822950642688 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1Ù<Îóß2Í- _wandbruntime‚3178  _runtime‚3178a[egÜ&ÎÖM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6643892420608 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6643892420608 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129911016 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚129630753114112 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚79745144 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50165872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚129630753114112 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6823851758080 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚129630770153472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50165870 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50165870 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 129911016 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 129901344 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21629893 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚129630770153472 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29730684 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6627718142586 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6627632339350 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29730682 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚133037321047738 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚126409585865792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51360321 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚136760486479872 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚79735474 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚129936763291648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392BÊgHá&ÏÛM H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 129901344 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21629893 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚129630770153472 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29730684 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6627718142586 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6627632339350 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29730682 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚133037321047738 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚126409585865792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51360321 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚136760486479872 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚79735474 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚129936763291648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6643892420608 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6643892420608 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129911016 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚129630753114112 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚79745144 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50165872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚129630753114112 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6823851758080 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚129630770153472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50165870 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50165870 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 129911016 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024ÚÃæ92Ð- _wandbruntime‚3178  _runtime‚3178;ÈÑý 1loss/SRqqt2/assignment_loss‚1.327541881418299 1loss/FBt2/assignment_loss‚0.19906002013616275 /loss/FRt1/detection_loss‚2.3530154212795455 /loss/FBt2/detection_loss‚0.8665485612343614 trainer/global_step‚7549 _runtime‚ 3178.39813986 epoch‚0 0loss/FRt2/assignment_loss‚2.9533904790878296 2loss/SRqqt1/assignment_loss‚0.9573182053318054 1loss/SRqqt1/detection_loss‚1.1621031113482754 .loss/FBt1/detection_loss‚2.949283431707458 /loss/FRt1/assignment_loss‚2.638954494876423 1loss/FBt1/assignment_loss‚0.09061238665982398 .loss/FRt2/detection_loss‚2.636102458915195 1loss/SRqqt2/detection_loss‚1.8407040594500446 &loss/total_loss‚19.974634511445224 % _timestamp‚1.7858134332082489e+09 _step‚603ÑØCÒ‚ % _timestamp‚1.7858134332082489e+09 _step‚603 1loss/SRqqt2/assignment_loss‚1.327541881418299 1loss/FBt2/assignment_loss‚0.19906002013616275 /loss/FRt1/detection_loss‚2.3530154212795455 /loss/FBt2/detection_loss‚0.8665485612343614 trainer/global_step‚7549 _runtime‚ 3178.39813986 epoch‚0 0loss/FRt2/assignment_loss‚2.9533904790878296 2loss/SRqqt1/assignment_loss‚0.9573182053318054 1loss/SRqqt1/detection_loss‚1.1621031113482754 .loss/FBt1/detection_loss‚2.949283431707458 /loss/FRt1/assignment_loss‚2.638954494876423 1loss/FBt1/assignment_loss‚0.09061238665982398 .loss/FRt2/detection_loss‚2.636102458915195 1loss/SRqqt2/detection_loss‚1.8407040594500446 &loss/total_loss‚19.974634511445224ÛÇrÛÓ‚(:… º³ÅӳŖƒ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚109.944' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚36.648 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0£Ì¢¬gÔ‚(:Ü  º³ÅÓõå݃% memory_percent‚41.239309310913086* proc.memory.percent‚17.172330617904663$ disk./.usageGB‚586.3584251403809 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1289.out‚0 disk.sda2.out‚ 109.10546875 disk.rbd1.in‚0 disk.drbd1285.in‚0 disk.drbd1356.out‚0 proc.cpu.threads‚29 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1965.out‚0 disk.drbd2111.out‚0 disk.drbd1817.in‚0 network.sent‚ 7.017967e+06 disk.drbd2118.out‚0 disk.nvme0n1.in‚9.59375 disk.drbd1828.in‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0 network.recv‚483173 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.dm-0.in‚9.59375 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 12.8037109375$ proc.memory.rssMB‚11254.05859375( disk./.usagePercent‚32.7922326917815 disk.dm-0.out‚664.7158203125 disk.drbd1552.out‚0 disk.drbd1931.out‚0 disk.drbd1817.out‚0 disk.drbd1285.out‚0# disk.nvme0n1.out‚664.7158203125 disk.drbd1584.out‚0 disk.sda.out‚ 109.10546875 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.drbd1673.in‚0 disk.drbd1938.out‚0 cpu‚13.762644712541153 disk.drbd1550.in‚0' proc.memory.availableMB‚ 38509.40625 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.sda2.in‚ 0.68359375 disk.sda.in‚ 0.68359375 disk.drbd2118.in‚0 ½jXÕ‚(:’ ɳÅÓ«ÆÀƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚121.825' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚40.608333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¶égcÖ‚(:Ø  ɳÅÓßÜʃ network.sent‚ 7.063001e+06 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.sda2.out‚ 109.41015625 disk.drbd1965.out‚0 disk.rbd0.out‚0* proc.memory.percent‚17.172330617904663 disk.sda1.out‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0" memory_percent‚41.238933801651 disk.drbd2118.in‚0# disk.nvme0n1.out‚664.9052734375 disk.sda.out‚ 109.41015625 disk.drbd1285.in‚0 disk.sda1.in‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.drbd1510.out‚0 disk.drbd1817.in‚0( disk./.usagePercent‚32.7922326917815 disk.drbd1289.in‚0 disk.drbd2041.out‚0 disk.drbd1289.out‚0 disk.drbd1356.out‚0 disk.sda.in‚ 0.68359375 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 disk.rbd1.out‚ 12.8359375* proc.memory.availableMB‚38509.65234375 proc.cpu.threads‚29$ disk./.usageGB‚586.3584251403809 disk.drbd1226.in‚0 disk.drbd1226.out‚0 disk.drbd1938.in‚0 disk.drbd1550.in‚0 disk.sda2.in‚ 0.68359375 disk.nvme0n1.in‚9.59375 disk.drbd1965.in‚0 disk.drbd2111.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1828.in‚0 disk.drbd1552.out‚0 disk.drbd2078.in‚0 disk.dm-0.out‚664.9052734375 disk.drbd1673.out‚0 disk.drbd1356.in‚0$ proc.memory.rssMB‚11254.05859375 cpu‚13.76220414199856 network.recv‚485704 disk.drbd1584.in‚0 disk.drbd1550.out‚0 disk.dm-0.in‚9.59375&n2×- _wandbruntime‚3198  _runtime‚3198ã¨î¼ÉØà ' lr-AdamW/pg1‚0.00019584284510184105 ' lr-AdamW/pg2‚0.00019584284510184105 trainer/global_step‚7599 % _timestamp‚1.7858134536034105e+09 _runtime‚3198.793345389 _step‚604Ž¸á2ÙÈ ' lr-AdamW/pg1‚0.00019584284510184105 'd¹G¤œ lr-AdamW/pg2‚0.00019584284510184105 trainer/global_step‚7599 % _timestamp‚1.7858134536034105e+09 _runtime‚3198.793345389 _step‚604Ü„òj2Ú- _wandbruntime‚3198  _runtime‚3198ì–{[Ð'ÛÊO @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50491379 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚133900497639108 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29923597 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚137158933437952 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚130471924501504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚137648102936576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50491377 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6687008936448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚130471941540864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚80252953 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚130471941540864 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚137158933437952 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚80262609 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51694913 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚137159037906944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6687096366080 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚127229854309472 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 $ _timestamp‚1.785813453604397e+09 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚130779912014848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 130753988 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚137647949455872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚133900600481592 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50491377 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚130471924501504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21771058 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚130779936918528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21771316 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6868037441024 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 _runtime‚ 3198.79465605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚137159037906944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50491379 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29923595 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚605 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚127229871348832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6687096366080 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6670643329636 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚80262609 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 130744330 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚80252953 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚7599 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6670729132760 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚127229871348832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6687096366080 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6670643329636 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚80262609 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 130744330 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚80252953 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚7599 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6670729132760 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚137158933437952 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚130471924501504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚137648102936576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50491377 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6687008936448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚130471941540864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚80252953 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚130471941540864 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚137158933437952 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚80262609 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51694913 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚137159037906944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6687096366080 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚127229854309472 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 $ _timestamp‚1.785813453604397e+09 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚130779912014848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 130753988 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚137647949455872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚133900600481592 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50491377 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚130471924501504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21771058 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚130779936918528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21771316 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6868037441024 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0ݯ8`2Ý- _wandbruntime‚3199  _runtime‚3199“¸£8Ü&ÞÖM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 % _timestamp‚1.7858134539528124e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6687888556032 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚133918115284650 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚80273146 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚137176979820544 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚137177084296704 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6687888556032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51701644 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚263 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚137666061561856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚127246611521152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50498022 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚130489108303872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚130797119131648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21774110 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚127246594481792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51701379 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6687975992832 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚137177084296704 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130771168 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚130489091264512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130771168 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29927534 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6868942430208 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50498020 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚80263476 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6869068902912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚3199.143097978 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚606 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚130489091264512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚137666212938240 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6671520802858 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130761496 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚80263476 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚265 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6671606606094 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚3199.143097978 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚606 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚130489091264512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚137666212938240 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6671520802858 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130761496 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMoni†@e“tor.on_train_batch_end/active.all.peak‚11887 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚80263476 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚265 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6671606606094 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚137177084296704 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6687888556032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51701644 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚263 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚137666061561856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 126472704 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚127246611521152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50498022 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚130489108303872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚130797119131648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21774110 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚127246594481792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51701379 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6687975992832 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚137177084296704 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130771168 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚130489091264512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 151376384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130771168 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548Þë®2à- _wandbruntime‚3199  _runtime‚3199ÍÛƒÓá 0loss/FRt2/assignment_loss‚3.0692444009440285 2loss/SRqqt2/assignment_loss‚1.0523727036779746 2loss/FBt1/assignment_loss‚0.010578151844900472 1loss/SRqqt2/detection_loss‚2.2092428728938103 /loss/FBt1/detection_loss‚2.3260374101144925 epoch‚0 3loss/FBt2/assignment_loss‚7.310583154870672e-05 /loss/FRt1/detection_loss‚2.4984589102791577 %loss/total_loss‚19.59385326789873 /loss/FRt1/assignment_loss‚2.756710424655821 1loss/SRqqt1/detection_loss‚1.0555071795779851 _runtime‚3199.143799464 _step‚607 2loss/SRqqt1/assignment_loss‚0.9795146651986626 -loss/FRt2/detection_loss‚2.82478147149086 /loss/FBt2/detection_loss‚0.8113319713894914 trainer/global_step‚7599 % _timestamp‚1.7858134539540026e+09%Z©_ ↠% _timestamp‚1.7858134539540026e+09 0loss/FRt2/assignment_loss‚3.0692444009440285 2loss/SRqqt2/assignment_loss‚1.0523727036779746 2loss/FBt1/assignment_loss‚0.010578151844900472 1loss/SRqqt2/detection_loss‚2.2092428728938103 /loss/FBt1/detection_loss‚2.3260374101144925 epoch‚0 3loss/FBt2/assignment_loss‚7.310583154870672e-05 /loss/FRt1/detection_loss‚2.4984589102791577 %loss/total_loss‚19.59385326789873 /loss/FRt1/assignment_loss‚2.756710424655821 1loss/SRqqt1/detection_loss‚1.0555071795779851 _runtime‚3199.143799464 _step‚607 2loss/SRqqt1/assignment_loss‚0.9795146651986626 -loss/FRt2/detection_loss‚2.82478147149086 /loss/FBt2/detection_loss‚0.8113319713894914 trainer/global_step‚7599ßü!L)‘ã‚(:† سÅӊŨƒ gpu.0.gpu‚31.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚40.0 gpu.0.powerWatts‚128.358' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚42.786 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0>cÁaä‚(:Ö  سÅÓ¡£Çƒ disk.sda.out‚ 109.734375 disk.sda2.in‚ 0.68359375 disk.sda2.out‚ 109.734375 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1584.in‚0$ memory_percent‚41.23902916908264 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 network.sent‚ 7.066034e+06$ disk./.usageGB‚586.3584251403809 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.drbd1510.out‚0 disk.drbd1931.in‚0 disk.sda.in‚ 0.68359375 disk.drbd1965.out‚0 proc.cpu.threads‚29 disk.nvme0n1.in‚9.59375 disk.drbd2041.in‚0 disk.drbd1550.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1817.in‚0 disk.drbd1584.out‚0 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.drbd2111.out‚0 cpu‚13.76172835548825 disk.rbd1.out‚ 12.8359375 disk.dm-0.in‚9.59375 disk.drbd2111.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1673.in‚0 disk.drbd2078.out‚0 disk.drbd1356.in‚0 disk.drbd2041.out‚0 disk.sda1.out‚0 disk.drbd2118.in‚0 network.recv‚486585 disk.drbd1938.out‚0 disk.drbd1226.in‚0 disk.drbd1285.in‚0 disk.dm-0.out‚665.1435546875( disk./.usagePercent‚32.7922326917815 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd1673.out‚0# disk.nvme0n1.out‚665.1435546875 disk.drbd1931.out‚0* proc.memory.availableMB‚38509.58984375 disk.rbd1.in‚0 disk.drbd1226.out‚0 disk.sda1.in‚0 disk.drbd1938.in‚0§#2å- _wandbruntime‚3219  _runtime‚3219FzÄ®Éæà _step‚608 ' lr-AdamW/pg1‚0.00019713145442610633 ' lr-AdamW/pg2‚0.00019713145442610633 trainer/global_step‚7649 % _timestamp‚1.7858134746011026e+09 _runtime‚3219.791037577z?÷ ÎçÈ ' lr-AdamW/pg1‚0.00019713145442610633 ' lr-AdamW/pg2‚0.00019713145442610633 trainer/global_step‚7649 % _timestamp‚1.7858134746011026e+09 _runtime‚3219.791037577 _step‚608à’Soc2è- _wandbruntime‚3219  _runtime‚3219bÕÎØÒ'éÌO SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6714531793144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 _runtime‚3219.795648249 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6731092501504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6731005071872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚138061267723776 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚131330262651904 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131604482 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30120445 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚131640267854848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50823527 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚131330262651904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50823529 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚80790611 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚128066862925472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30120447 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚80780955 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50823529 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚7649 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚138061267723776 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚138061372192768 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚131330279691264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52034959 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚134781497561100 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚80790611 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21914767 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858134746020303e+09 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚138061267723776 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚131330262651904 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131604482 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30120445 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚131640267854848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50823527 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚131330262651904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50823529 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚80790611 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚128066862925472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30120447 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚80780955 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50823529 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚7649 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚138061267723776 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚138061372192768 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚131330279691264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52034959 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚134781497561100 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚80790611 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21914767 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 % _timestamp‚1.7858134746020303e+09 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚7649 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚138079314106368 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚80801148 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131621648 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 131631320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131621648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30124382 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21917637 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52041770 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50830170 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚138079418582528 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚131347429414912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚138571789998080 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6914290122752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6731972128256 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 131631320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50830172 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6731884691456 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚131347446454272 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚128083620137152 _step‚610 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚80791478 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6715409266366 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032:ÂE˜á&íÛM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 131631320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131621648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30124382 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21917637 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52041770 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50830170 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚138079418582528 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚131347429414912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚138571789998080 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6914290122752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6731972128256 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 131631320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50830172 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6731884691456 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚131347446454272 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚128083620137152 _step‚610 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚80791478 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6715409266366 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚131347429414912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 % _timestamp‚1.7858134749540768e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚134799012364158 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52042021 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21917388 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚138571630233088 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚80791478 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚131657499875328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6715495069602 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6731884691456 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚138079418582528 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚3220.144349657 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50830172 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚7649 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚138079314106368 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚80801148 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131621648 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236â 9'2î- _wandbruntime‚3220  _runtime‚3220Ö0{ïý 0loss/FRt1/assignment_loss‚2.6260031132137076 2loss/SRqqt1/assignment_loss‚1.1257646046171006 1loss/SRqqt2/assignment_loss‚1.140535077503459 2loss/FBt1/assignment_loss‚0.060888919687325446 1loss/FBt2/assignment_loss‚0.04941467718579519 epoch‚0 .loss/FRt2/detection_loss‚2.676588026028645 1loss/SRqqt1/detection_loss‚1.1425354576186977 /loss/FBt1/detection_loss‚2.0357419669628145 % _timestamp‚1.7858134749556048e+09 /loss/FRt2/assignment_loss‚2.969518976875498 0loss/SRqqt2/detection_loss‚2.042230887846513 /loss/FBt2/detection_loss‚0.8373644809903843 &loss/total_loss‚19.117547096325794 _runtime‚3220.145355867 -loss/FRt1/detection_loss‚2.41096090779585 trainer/global_step‚7649 _step‚611pxð‚ .loss/FRt2/detection_loss‚2.676588026028645 1loss/SRqqt1/detection_loss‚1.1425354576186977 /loss/FBt1/detection_loss‚2.0357419669628145 % _timestamp‚1.7858134749556048e+09 /loss/FRt2/assignment_loss‚2.969518976875498 0loss/SRqqt2/detection_loss‚2.042230887846513 /loss/FBt2/detection_loss‚0.8373644809903843 &loss/total_loss‚19.117547096325794 _runtime‚3220.145355867 -loss/FRt1/detection_loss‚2.41096090779585 trainer/global_step‚7649 _step‚611 0loss/FRt1/assignment_loss‚2.6260031132137076 2loss/SRqqt1/assignment_loss‚1.1257646046171006 1loss/SRqqt2/assignment_loss‚1.140535077503459 2loss/FBt1/assignment_loss‚0.060888919687325446 1loss/FBt2/assignment_loss‚0.04941467718579519 epoch‚0ãØÖžñ‚(:’ ç³ÅÓº­¬ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.092' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.364000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0e¢Ëbò‚(:×  ç³ÅÓ¨£âƒ disk.sda1.in‚0 disk.rbd1.out‚ 12.9921875% memory_percent‚41.239458322525024 disk.drbd1226.out‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd1552.out‚0 disk.drbd1817.in‚0 disk.drbd1828.out‚0 disk.dm-0.out‚ 665.3515625 disk.drbd1673.in‚0 disk.nvme0n1.out‚ 665.3515625$ disk./.usageGB‚586.3584289550781 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd1938.out‚0 disk.nvme0n1.in‚9.59375 proc.cpu.threads‚29 disk.drbd1931.in‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.sda2.out‚ 110.04296875 disk.drbd1226.in‚0 disk.drbd1965.in‚0 cpu‚13.761368882397122 disk.drbd1584.in‚0 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.drbd1356.out‚0 disk.drbd2078.out‚0 disk.sda2.in‚ 0.68359375* proc.memory.percent‚17.172330617904663 disk.drbd2041.out‚0 disk.rbd0.in‚0* proc.memory.availableMB‚38509.30859375$ proc.memory.rssMB‚11254.05859375 network.sent‚ 7.111267e+06) disk./.usagePercent‚32.79223290511935 disk.drbd1356.in‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd2078.in‚0 disk.drbd2111.out‚0 disk.drbd1285.out‚0 network.recv‚489198 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.drbd1828.in‚0 disk.dm-0.in‚9.59375 disk.drbd2118.in‚0 disk.sda.in‚ 0.68359375 disk.sda.out‚ 110.04296875 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1289.out‚0¬ß ›ó‚(: ö³ÅÓõ‡Žƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.857' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.28566666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0yk†ê\ô‚(:Ñ  ö³ÅÓªÕ܃$ memory_percent‚41.24000072479248) disk./.usagePercent‚32.79223311845719 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.drbd1356.out‚0 disk.drbd1510.in‚0 disk.drbd1938.out‚0 disk.drbd2041.in‚0 disk.drbd1550.out‚0 disk.dm-0.in‚9.59375 disk.drbd1552.out‚0 disk.drbd2041.out‚0 disk.sda.in‚ 0.68359375 disk.sda.out‚ 110.34765625 disk.drbd2078.out‚0 disk.dm-0.out‚ 665.609375 disk.drbd2118.in‚0 cpu‚13.76093920122015 network.sent‚ 7.156352e+06 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.sda2.out‚ 110.34765625( proc.memory.availableMB‚ 38508.953125 disk.drbd1817.in‚0 disk.rbd1.out‚ 12.9921875 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1289.out‚0 disk.drbd1550.in‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.drbd1356.in‚0 disk.drbd1931.in‚0 disk.sda2.in‚ 0.68359375 disk.nvme0n1.out‚ 665.609375 network.recv‚491633 disk.drbd1285.out‚0 disk.drbd1584.in‚0 disk.drbd1965.in‚0 disk.drbd1285.in‚0 disk.drbd1584.out‚0$ disk./.usageGB‚586.3584327697754 disk.drbd1965.out‚0 disk.rbd1.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1226.in‚0 disk.sda1.out‚0* proc.memory.percent‚17.172330617904663 proc.cpu.threads‚29 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚9.59375 disk.drbd1931.out‚0 disk.drbd1673.out‚0 disk.drbd1510.out‚0 disk.sda1.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0]]¨ô2õ- _wandbruntime‚3240  _runtime‚3240Ü5 öÉöà ' lr-AdamW/pg1‚0.00019842006375037167 ' lr-AdamW/pg2‚0.00019842006375037167 trainer/global_step‚7699 % _timestamp‚1.7858134951197517e+09 _runtime‚3240.309685579 _step‚612*Î CÎ÷È ' lr-AdamW/pg2‚0.00019842006375037167 trainer/global_step‚7699 % _timestamp‚1.7858134951197517e+09 _runtime‚3240.309685579 _step‚612 ' lr-AdamW/pg1‚0.00019842006375037167äß©äŠ2ø- _wandbruntime‚3240  _runtime‚3240yZý±ùÌO ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6775001207296 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30317295 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81308957 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚128903871541472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6775088636928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚253 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6958438375936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚139459219745792 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52375535 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135662394640608 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚132500648598528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6958571147264 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51155677 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚138963706478592 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22058238 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22057987 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6758506059776 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚132188617841664 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858134951206589e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚128903888580832 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51155679 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52375282 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51155679 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚132500623694848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚132188600802304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚81318613 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚135662291798124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚138963602009600 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 132474292 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚138963706478592 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51155677 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30317297 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚7699 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚138963602009600 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6775001207296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6758420256652 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81318613 _runtime‚3240.313651714 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 132464634 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚81308957 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚132188617841664_„×'úÑO ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚132188617841664 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858134951206589e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚128903888580832 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51155679 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52375282 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51155679 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚132500623694848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚132188600802304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚81318613 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚135662291798124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚138963602009600 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 132474292 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚138963706478592 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51155677 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30317297 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚7699 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚138963602009600 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6775001207296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6758420256652 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81318613 _runtime‚3240.313651714 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 132464634 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚81308957 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚132188617841664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6775001207296 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30317295 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81308957 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚128903871541472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6775088636928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚253 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6958438375936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚139459219745792 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52375535 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135662394640608 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚132500648598528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6958571147264 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51155677 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚138963706478592 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22058238 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22057987 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6758506059776 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121å«Þ}À2û- _wandbruntime‚3240  _runtime‚3240uQÀ[Ú&üÔM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132491472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚138981752868352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22061145 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6959475449344 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _step‚614 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132491472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52382124 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51162320 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚81319480 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚132517830811648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6759297729874 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6775968263680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6759383533110 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚128920611713792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _runtime‚ 3240.65921164 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚138981648392192 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚135679909443666 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚81329150 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚81319480 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚139477331164672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚132205767565312 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30321234 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚132517855715328 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7699 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6775968263680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚132205784604672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚81329150 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚132205784604672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6775880826880 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 132481800 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51162322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51162320 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130êÞ~Ù#ýÙM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51162320 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132491472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚138981752868352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22061145 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6959475449344 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _step‚614 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132491472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52382124 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51162320 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚81319480 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚132517830811648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6759297729874 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6775968263680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6759383533110 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚128920611713792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _runtime‚ 3240.65921164 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚138981648392192 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚135679909443666 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚81329150 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚81319480 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚139477331164672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚132205767565312 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30321234 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚132517855715328 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7699 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6775968263680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚132205784604672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚81329150 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.al¾(x’l.current‚ 163959296 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚132205784604672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6775880826880 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 132481800 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51162322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2æ7GÖ2þ- _wandbruntime‚3240  _runtime‚3240×Vwoÿ‚ trainer/global_step‚7699 0loss/FRt2/assignment_loss‚3.1299052658215376 .loss/FRt2/detection_loss‚2.828087130063017 1loss/SRqqt2/detection_loss‚1.9760530151737232 .loss/FBt1/detection_loss‚1.247102223150432 0loss/FRt1/assignment_loss‚2.0347896608305565 epoch‚0 _step‚615 1loss/FBt1/assignment_loss‚0.07239341975177922 1loss/SRqqt1/detection_loss‚1.0320755312193746 &loss/total_loss‚18.174460843724336 % _timestamp‚1.7858134954700987e+09 _runtime‚3240.659889745 2loss/SRqqt1/assignment_loss‚1.0394385693186563 2loss/SRqqt2/assignment_loss‚1.2047112628545922 3loss/FBt2/assignment_loss‚0.0061798373588184165 /loss/FRt1/detection_loss‚2.3882199722675628 /loss/FBt2/detection_loss‚1.2155049559142854ƒ(¸S €‡ trainer/global_step‚7699 0loss/FRt2/assignment_loss‚3.1299052658215376 .loss/FRt2/detection_loss‚2.828087130063017 1loss/SRqqt2/detection_loss‚1.9760530151737232 .loss/FBt1/detection_loss‚1.247102223150432 0loss/FRt1/assignment_loss‚2.0347896608305565 epoch‚0 _step‚615 1loss/FBt1/assignment_loss‚0.07239341975177922 1loss/SRqqt1/detection_loss‚1.0320755312193746 &loss/total_loss‚18.174460843724336 % _timestamp‚1.7858134954700987e+09 _runtime‚3240.659889745 2loss/SRqqt1/assignment_loss‚1.0394385693186563 2loss/SRqqt2/assignment_loss‚1.2047112628545922 3loss/FBt2/assignment_loss‚0.0061798373588184165 /loss/FRt1/detection_loss‚2.3882199722675628 /loss/FBt2/detection_loss‚1.2155049559142854çŽÝRq‚(:’ …´ÅÓ¨—¢ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚107.162' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚35.720666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0å–ÓXf‚‚(:Û  …´ÅÓí±Øƒ disk.drbd1356.in‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 110.6328125 disk.rbd1.in‚0 disk.rbd0.in‚0$ memory_percent‚41.24014377593994 disk.drbd1931.in‚0 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.drbd1828.out‚0 network.sent‚ 7.15938e+06 disk.drbd1552.in‚0 disk.sda.out‚ 110.6328125$ disk./.usageGB‚586.3584327697754 disk.drbd1584.in‚0! disk.dm-0.out‚665.83544921875 network.recv‚492547) disk./.usagePercent‚32.79223311845719 disk.drbd2111.in‚0 disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1510.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd2111.out‚0 disk.rbd1.out‚ 13.0244140625 disk.drbd1285.in‚0 disk.dm-0.in‚9.59375 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.sda2.in‚ 0.68359375 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.sda1.in‚0 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0* proc.memory.percent‚17.172330617904663 disk.nvme0n1.in‚9.59375 disk.drbd1828.in‚0 proc.cpu.threads‚29 disk.sda1.out‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.rbd0.out‚0 cpu‚13.76032494876804 disk.drbd2118.out‚0$ disk.nvme0n1.out‚665.83544921875 disk.sda.in‚ 0.68359375( proc.memory.availableMB‚ 38508.859375 disk.drbd1289.in‚0 disk.drbd1673.in‚0DÀh2ƒ- _wandbruntime‚3261  _runtime‚3261ˆ6˜É„à ' lr-AdamW/pg1‚0.00019970867307463695 ' lr-AdamW/pg2‚0.00019970867307463695 trainer/global_step‚7749 % _timestamp‚1.7858135163351207e+09 _runtime‚3261.525063549 _step‚616ÏeÎ…È ' lr-AdamW/pg1‚0.00019970867307463695 ' lr-AdamW/pg2‚0.00019970867307463695 trainer/global_step‚7749 % _timestamp‚1.7858135163351207e+09 _runtime‚3261.525063549 _step‚616èãÝk·2†- _wandbruntime‚3261  _runtime‚3261šuÒzÒ'‡ÌO G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚129740897196832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81836959 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6818997342720 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6818997342720 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30514145 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7003652268032 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚81846615 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858135163360386e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚133046938952704 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚139866040764416 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6819084772352 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _step‚617 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚139865936295424 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6802308720160 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚133046955992064 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 133334444 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51487829 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52715409 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22201264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚133361004438528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚133360979534848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚140364797866496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚140364631802880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6802394523284 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚129740880157472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚81836959 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22201519 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51487827 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚139865936295424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚133046938952704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52715666 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81846615 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚139865936295424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚133046938952704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52715666 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81846615 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚129740897196832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81836959 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6818997342720 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6818997342720 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30514145 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7003652268032 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚81846615 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858135163360386e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚133046938952704 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚139866040764416 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6819084772352 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _step‚617 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚139865936295424 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6802308720160 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚133046955992064 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 133334444 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51487829 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52715409 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22201264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚133361004438528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚133360979534848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚140364797866496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚140364631802880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6802394523284 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚129740880157472éqæ 2‰- _wandbruntime‚3261  _runtime‚3261I Y§xŠÖM MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7004553882112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚140382740533760 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚133064122755072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51494470 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚133378211555328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 _runtime‚3261.883012636 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚139883982678016 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6819876962304 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51494472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6803271996618 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51494472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52722379 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30518084 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚133064105715712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52722150 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22204295 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133351624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6819964399104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚139884087154176 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7004684549120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133351624 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚133064105715712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 133341952 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚81847482 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22204068 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6819876962304 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚227 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚140382896104448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚129757637369152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚136560909365770 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚129757620329792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7858135166927514e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6803186193382 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51494470 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0’ýºŸá&‹ÛM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚81857152 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚229 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚136560909365770 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚129757620329792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7858135166927514e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6803186193382 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51494470 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7004553882112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚140382740533760 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚133064122755072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51494470 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚133378211555328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 _runtime‚3261.883012636 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚139883982678016 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6819876962304 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51494472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6803271996618 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51494472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52722379 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30518084 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚133064105715712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52722150 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22204295 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133351624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6819964399104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚139884087154176 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7004684549120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133351624 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚133064105715712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 133341952 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚81847482 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22204068 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6819876962304 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚227 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚140382896104448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚129757637369152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚140768270581248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚137424085957140 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚82364961 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚133905277103104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51819979 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚140768375050240 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30710997 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚141270339774464 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 trainer/global_step‚7799 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚140768270581248 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚82374617 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 # _timestamp‚1.78581353713395e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 134194596 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30710995 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7048842530304 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚130577888773472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 134184938 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53056117 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚134221360278528 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6863080907776 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6863080907776 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚133905277103104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22345122 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚141270177905152 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51819977 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6863080907776 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6863080907776 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚133905277103104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22345122 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚141270177905152 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51819977 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚140768270581248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚137424085957140 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚82364961 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚133905277103104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51819979 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚140768375050240 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/ì¤ €segment.all.allocated‚251 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30710997 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚141270339774464 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 trainer/global_step‚7799 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚140768270581248 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚82374617 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 # _timestamp‚1.78581353713395e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 134194596 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30710995 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7048842530304 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚130577888773472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 134184938íÃ`Û2—- _wandbruntime‚3282  _runtime‚3282G^K£Ü&˜ÖM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51826620 % _timestamp‚1.7858135374856725e+09 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7049744742400 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚133922460905472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30714932 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _step‚622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚130594628945792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚82375484 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚141288287234048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6863960534528 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 134202104 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30714934 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚140786421440000 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚137441703602682 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6863873097728 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7049879603712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚141288446999040 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6847074656890 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 134202104 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6863960534528 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6847160460126 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51826622 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53063199 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53062940 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚82385154 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚130594645985152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚133922443866112 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134211776 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 134211776 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7049744742400 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚133922460905472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30714932 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _step‚622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚130594628945792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚82375484 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚141288287234048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6863960534528 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 134202104 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30714934 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚140786421440000 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚137441703602682 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6863873097728 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7049879603712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚141288446999040 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6847074656890 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 134202104 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6863960534528 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6847160460126 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51826622 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53063199 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53062940 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚82385154 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚130594645985152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚133922443866112 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134211776 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 134211776 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E(ߥÊO TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52152129 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚134763632292864 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚131414897389472 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚141670709336064 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚82892963 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6906989613568 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52152127 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30907847 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚13830508587913÷Gñ2 _runtime‚ 3303.11663632 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚141670604867072 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚134763615253504 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6907077043200 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚82892963 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53397120 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚82902619 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135054748 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚82902619 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 $ _timestamp‚1.785813557923248e+09 _step‚625 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22489535 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52152129 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚141670709336064 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6907077043200 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7094013996544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135045090 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚134763615253504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚141670709336064 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6907077043200 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7094013996544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135045090 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚134763615253504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚131414897389472 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚141670709336064 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚82892963 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6906989613568 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52152127 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30907847 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚138305085879132 _runtime‚ 3303.11663632 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚141670604867072 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚134763615253504 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6907077043200 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚82892963 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53397120 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024ñ\÷p@2§- _wandbruntime‚3303  _runtime‚3303ÆTû+Ü&¨ÖM C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858135582766232e+09 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚3303.466974348 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52158770 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚131431637561792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚7849 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52158772 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚135098923235328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚135098898331648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6907869233152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚138322600682190 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _step‚626 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 135071928 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚131431654601152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6907869233152 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53404230 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135062256 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚141688755725824 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚134780799055872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚142193810722816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6890963120398 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚134780782016512 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚82903486 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52158770 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚82913156 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7095051446784 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53403975 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52158772 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚134780782016512 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135071928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30911784 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚82913156 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30911782 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚224924463Øq ©ÛM WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚138322703524786 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6891048923634 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚82903486 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚141688651249664 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22492193 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6907956669952 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚142193974682112 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6907956669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 135062256 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53403975 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52158772 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚134780782016512 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135071928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30911784 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚82913156 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30911782 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22492446 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858135582766232e+09 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚3303.466974348 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52158770 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚131431637561792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚7849 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52158772 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚135098923235328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚135098898331648 4.DeviceStaÒŸ>äØtsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6907869233152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚138322600682190 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 _step‚626 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 135071928 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚131431654601152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6907869233152 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53404230 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135062256 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚141688755725824 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚134780799055872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚142193810722816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6890963120398 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚134780782016512 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚82903486 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52158770 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚82913156 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7095051446784 F2ª- _wandbruntime‚3303  _runtime‚3303FLK«ÿ &loss/total_loss‚18.176683551323755 2loss/FBt2/assignment_loss‚0.008321372427121885 /loss/FRt2/detection_loss‚2.7291579985920387 % _timestamp‚1.7858135582779222e+09 _step‚627 .loss/FRt1/detection_loss‚2.473982025755615 trainer/global_step‚7849 _runtime‚3303.467729304 /loss/FBt2/detection_loss‚0.8721790947260913 epoch‚0 0loss/FRt1/assignment_loss‚2.3731884374851133 /loss/FRt2/assignment_loss‚2.946880161007748 1loss/SRqqt1/assignment_loss‚0.915061785839498 1loss/SRqqt1/detection_loss‚0.9799710356861733 0loss/SRqqt2/detection_loss‚2.014476828963349 .loss/FBt1/detection_loss‚1.664634665904137 2loss/SRqqt2/assignment_loss‚1.1686852384162576 2loss/FBt1/assignment_loss‚0.030144906520613055©lÆ ¬„ .loss/FBt1/detection_loss‚1.664634665904137 2loss/SRqqt2/assignment_loss‚1.1686852384162576 2loss/FBt1/assignment_loss‚0.030144906520613055 &loss/total_loss‚18.176683551323755 2loss/FBt2/assignment_loss‚0.008321372427121885 /loss/FRt2/detection_loss‚2.7291579985920387 % _timestamp‚1.7858135582779222e+09 _step‚627 .loss/FRt1/detection_loss‚2.473982025755615 trainer/global_step‚7849 _runtime‚3303.467729304 /loss/FBt2/detection_loss‚0.8721790947260913 epoch‚0 0loss/FRt1/assignment_loss‚2.3731884374851133 /loss/FRt2/assignment_loss‚2.946880161007748 1loss/SRqqt1/assignment_loss‚0.915061785839498 1loss/SRqqt1/detection_loss‚0.9799710356861733 0loss/SRqqt2/detection_loss‚2.014476828963349ólç¬Ã‘­‚(:† Á´ÅÓÛ’ø‚ gpu.0.gpu‚33.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚100.644' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚33.548 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ö¯ÒWq®‚(:æ  Á´ÅÓ榼ƒ# disk.nvme0n1.out‚685.4638671875 disk.drbd1356.out‚0 disk.rbd0.in‚0 disk.drbd1817.out‚0 disk.sda2.out‚ 116.86328125 disk.rbd1.in‚0* proc.memory.percent‚17.172330617904663 cpu‚13.75866328487798$ memory_percent‚41.24080538749695 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd1285.in‚0 disk.nvme0n1.in‚ 10.54296875 network.recv‚500482 disk.drbd2111.out‚0 proc.cpu.threads‚29 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.rbd0.out‚0 disk.drbd1510.out‚0 disk.sda.out‚ 116.86328125 disk.drbd2078.in‚0 disk.sda1.in‚0 disk.drbd1828.out‚0 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.drbd1938.in‚0 disk.rbd1.out‚ 13.1884765625 disk.drbd1285.out‚0* proc.memory.availableMB‚38508.42578125 disk.drbd1828.in‚0 network.sent‚ 7.297729e+06 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.dm-0.out‚685.4638671875 disk.drbd2118.in‚0 disk.drbd1938.out‚0 disk.dm-0.in‚ 10.54296875 disk.drbd1673.in‚0) disk./.usagePercent‚32.79243642941951 disk.drbd1550.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.sda2.in‚ 9.17578125 disk.sda.in‚ 9.17578125 disk.drbd1673.out‚0$ proc.memory.rssMB‚11254.05859375$ disk./.usageGB‚586.3620681762695 disk.drbd1931.in‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0ÊEH2¯- _wandbruntime‚3324  _runtime‚3324‹c©ŠÈ°Â ' lr-AdamW/pg1‚0.00020357450104743286 ' lr-AdamW/pg2‚0.00020357450104743286 trainer/global_step‚7899 % _timestamp‚1.7858135790969203e+09 _runtime‚ 3324.28685467 _step‚6285¦¢Í±Ç % _timestamp‚1.7858135790969203e+09 _runtime‚ 3324.28685467 _step‚628 ' lr-AdamW/pg1‚0.00020357450104743286 ' lr-AdamW/pg2‚0.00020357450104743286 trainer/global_step‚7899ôrⶮ2²- _wandbruntime‚3324  _runtime‚3324-ãÒ'³ÌO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52484279 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22632035 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚135621953403904 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135914900 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31104697 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚142573043621888 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6950985748992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52484277 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53736975 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7139147560448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _step‚629 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52484279 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6933974110684 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚132251906005472 trainer/global_step‚7899 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135905242 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚139185982958640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚142572939152896 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22632278 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚143081194615296 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚135621970443264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚83420965 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚135621970443264 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚135621953403904 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52484277 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6934059913808 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚83430621 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚132251923044832 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚142573043621888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858135790978115e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31104695 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚135942071958528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚135942047054848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚142572939152896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53736730 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚142572939152896 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22632278 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚143081194615296 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚245 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚135621970443264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚83420965 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚135621970443264 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚135621953403904 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52484277 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6934059913808 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚83430621 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚132251923044832 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚142573043621888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858135790978115e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31104695 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚135942071958528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚135942047054848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚142572939152896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53736730 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52484277 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53736975 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7139147560448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _step‚629 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52484279 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6933974110684 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚132251906005472 trainer/global_step‚7899 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135905242õ‰#Ü´2µ- _wandbruntime‚3324  _runtime‚3324M&)¯ ¶ÖM EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚135639120166912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚143099303673856 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135932080 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6934851583906 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚132268663217152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6951865368576 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53743360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7140049502208 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6934937387142 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6951952805376 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52490920 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚135959254171648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚139203600604294 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31108632 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22634985 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52490922 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52490920 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 _step‚630 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚135639137206272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚7899 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135922408 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6951952805376 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22634728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚83441158 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53743619 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚139203497761698 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚135639137206272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6951865368576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚83431488 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚135959279075328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚142591090011648 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858135794471486e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚142590985535488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚132268646177792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚83431488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31108634 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 135932080 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0]Ô%]á&·ÛM D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135922408 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6951952805376 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22634728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚83441158 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53743619 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚139203497761698 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚135639137206272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6951865368576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚83431488 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚135959279075328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚142591090011648 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858135794471486e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚142590985535488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚132268646177792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚83431488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31108634 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 135932080 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚135639120166912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚143099303673856 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135932080 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6934851583906 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚132268663217152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6951865368576 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53743360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7140049502208 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6934937387142 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6951952805376 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52490920 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚135959254171648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚139203600604294 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31108632 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22634985 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52490922 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52490920 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 _step‚630 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚135639137206272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚7899ö ×Ê2¸- _wandbruntime‚3324  _runtime‚3324+¿¢W¹û /loss/FRt1/detection_loss‚2.5205707851844497 .loss/FBt1/detection_loss‚2.138183702137196 /loss/FBt2/detection_loss‚0.7833590866648592 $ _timestamp‚1.785813579448464e+09 _runtime‚ 3324.63824597 /loss/FRt1/assignment_loss‚2.324449343771874 2loss/SRqqt2/assignment_loss‚0.9183421662077308 0loss/FBt1/assignment_loss‚0.0440801346886488 1loss/SRqqt1/detection_loss‚0.8609270610039433 1loss/SRqqt2/detection_loss‚2.3809559656927983 _step‚631 0loss/FRt2/assignment_loss‚2.7038093209266663 1loss/FBt2/assignment_loss‚0.00982021569511593 .loss/FRt2/detection_loss‚2.563782426132553 %loss/total_loss‚18.19473485627956 trainer/global_step‚7899 2loss/SRqqt1/assignment_loss‚0.9464546481737246 epoch‚0DþѺ€ 2loss/SRqqt1/assignment_loss‚0.9464546481737246 epoch‚0 /loss/FRt1/detection_loss‚2.5205707851844497 .loss/FBt1/detection_loss‚2.138183702137196 /loss/FBt2/detection_loss‚0.7833590866648592 $ _timestamp‚1.785813579448464e+09 _runtime‚ 3324.63824597 /loss/FRt1/assignment_loss‚2.324449343771874 2loss/SRqqt2/assignment_loss‚0.9183421662077308 0loss/FBt1/assignment_loss‚0.0440801346886488 1loss/SRqqt1/detection_loss‚0.8609270610039433 1loss/SRqqt2/detection_loss‚2.3809559656927983 _step‚631 0loss/FRt2/assignment_loss‚2.7038093209266663 1loss/FBt2/assignment_loss‚0.00982021569511593 .loss/FRt2/detection_loss‚2.563782426132553 %loss/total_loss‚18.19473485627956 trainer/global_step‚7899÷8³%ör»‚(:ç  дÅÓ¸Î׃ disk.drbd1510.out‚0 disk.rbd1.in‚0 disk.drbd1550.in‚0 disk.rbd0.in‚0 network.recv‚502810* disk./.usagePercent‚32.792442829554744 disk.drbd1289.out‚0 disk.nvme0n1.in‚ 10.54296875 disk.drbd1356.out‚0 disk.drbd1584.out‚0 disk.rbd0.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1673.in‚0* proc.memory.percent‚17.172330617904663 disk.dm-0.out‚685.8134765625 disk.drbd1938.out‚0 disk.rbd1.out‚ 13.1884765625 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.drbd2041.in‚0 disk.sda.in‚ 9.17578125 proc.cpu.threads‚29 network.sent‚ 7.342822e+06 disk.dm-0.in‚ 10.54296875 disk.drbd1289.in‚0 disk.drbd2111.out‚0 disk.drbd1552.out‚0 disk.drbd1584.in‚0 disk.drbd1285.out‚0 disk.drbd1510.in‚0 disk.drbd2078.out‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.drbd2118.in‚0 disk.drbd1828.in‚0 disk.sda2.out‚ 117.27734375 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.drbd2118.out‚0# disk.nvme0n1.out‚685.8134765625 disk.drbd1931.in‚0 disk.drbd1965.in‚0 cpu‚13.758144273832224 disk.drbd1817.in‚0 disk.drbd1226.out‚0 disk.sda2.in‚ 9.17578125 disk.drbd1552.in‚0 disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.sda.out‚ 117.27734375 disk.drbd1226.in‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.drbd1931.out‚0 disk.sda1.in‚0 disk.drbd1356.in‚0$ memory_percent‚41.24048948287964) proc.memory.availableMB‚ 38508.6328125$ disk./.usageGB‚586.3621826171875Q*j‘¼‚(:† дÅÓã±àƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.425' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚42.475 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0×ì\µ2½- _wandbruntime‚3344  _runtime‚3344ã ‡ôɾà trainer/global_step‚7949 % _timestamp‚1.7858135996067643e+09 _runtime‚3344.798621658 _step‚632 ' lr-AdamW/pg1‚0.00020486311037169817 ' lr-AdamW/pg2‚0.00020486311037169817¬0#Î¿È _step‚632 ' lr-AdamW/pg1‚0.00020486311037169817 ' lr-AdamW/pg2‚0.00020486311037169817 trainer/global_step‚7949 % _timestamp‚1.7858135996067643e+09 _runtime‚3344.798621658ø¤Ͷ2À- _wandbruntime‚3344  _runtime‚3344-&zÕÒ'ÁÌO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6977862574192 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚143475377907712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 136775052 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31301547 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚143986728436224 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6995069314048 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚136480291554304 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚143475377907712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚133088914621472 _step‚633 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚143475273438720 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52816429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6994981884416 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚83948967 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚83958623 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52816429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7184454118400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚3344.798699916 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚83958623 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚140066880038148 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6977948377316 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52816427 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22774330 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚136802402894848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚136480308593664 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22774578 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚136480291554304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31301545 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 % _timestamp‚1.7858135996074395e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚143475273438720 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 136765394 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚143475273438720 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52816429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6994981884416 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚83948967 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.alloca¥ ìùI$ted‚83958623 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52816429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7184454118400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚3344.798699916 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚83958623 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚140066880038148 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6977948377316 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52816427 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22774330 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚136802402894848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚136480308593664 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22774578 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚136480291554304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31301545 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 % _timestamp‚1.7858135996074395e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚143475273438720 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 136765394 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚143475377907712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130ù<ޛÂ(: ß´ÅӪ䗃 gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚128.05' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.68333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‘Â{ÃqÄ‚(:æ  ß´ÅÓÞÃàƒ) disk./.usagePercent‚32.79256571215107 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd2078.in‚0 disk.drbd1510.in‚0 disk.drbd1938.in‚0 disk.drbd1584.out‚0 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.sda2.in‚ 9.17578125 disk.drbd2041.out‚0 network.recv‚505075 disk.drbd1965.in‚0 disk.sda2.out‚ 119.71484375 disk.drbd1584.in‚0$ disk./.usageGB‚586.3643798828125 disk.drbd1285.in‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.sda.out‚ 119.71484375 disk.drbd1828.out‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0" disk.nvme0n1.out‚ 694.576171875 disk.drbd1285.out‚0 disk.sda1.out‚0 disk.rbd1.out‚ 13.2509765625 disk.nvme0n1.in‚ 10.54296875 disk.drbd1931.in‚0* proc.memory.availableMB‚38508.35546875$ proc.memory.rssMB‚11254.05859375 disk.rbd0.out‚0 disk.drbd1828.in‚0 disk.drbd1938.out‚0 disk.drbd1356.in‚0 disk.drbd2078.out‚0 disk.dm-0.out‚ 694.576171875 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0% memory_percent‚41.240912675857544 disk.drbd1510.out‚0 disk.sda1.in‚0 network.sent‚ 7.387864e+06 disk.drbd1817.out‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.drbd1931.out‚0 disk.drbd1356.out‚0 disk.sda.in‚ 9.17578125 disk.drbd1289.out‚0 disk.drbd2041.in‚0 proc.cpu.threads‚29 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.dm-0.in‚ 10.54296875* proc.memory.percent‚17.172330617904663 cpu‚13.757667718616517è“{2Å- _wandbruntime‚3345  _runtime‚3345B-æÜ&ÆÖM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52823070 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22777468 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22777207 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 136792232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚144004831703040 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚143493424297472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚83959490 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6995861504000 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6995861504000 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚634 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6978740047414 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54082952 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚144004995662336 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚136819634915328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7185221691392 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6978825850650 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚83969160 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6995948940800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 trainer/global_step‚7949 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136792232 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚136497475356672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7858135999567015e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54082689 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7185360747008 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136792232 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚136497475356672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7858135999567015e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54082689 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7185360747008 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 136792232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚144004831703040 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚143493424297472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚83959490 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6995861504000 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6995861504000 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 163959296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚634 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6978740047414 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54082952 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚144004995662336 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚136819634915328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7185221691392 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6978825850650 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚83969160 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251újgJ2È- _wandbruntime‚3345  _runtime‚3345´·¡pÉý _step‚635 epoch‚0 2loss/SRqqt2/assignment_loss‚0.7654325505970297 0loss/FBt2/assignment_loss‚0.0325112310441322 /loss/FRt1/detection_loss‚2.2957426999943165 1loss/SRqqt2/detection_loss‚2.0326377645739027 trainer/global_step‚7949 % _timestamp‚1.7858135999579136e+09 _runtime‚3345.147693251 0loss/FRt1/assignment_loss‚2.6208092494855952 2loss/SRqqt1/assignment_loss‚0.9637055080959552 /loss/FBt1/detection_loss‚2.5019248273223638 .loss/FRt2/assignment_loss‚2.78193399310112 0loss/FBt1/assignment_loss‚0.1993013911766866 .loss/FRt2/detection_loss‚2.593634119523423 1loss/SRqqt1/detection_loss‚0.9487513584723131 /loss/FBt2/detection_loss‚0.9486428781102101 &loss/total_loss‚18.685027571497045.ÔnËHÊ‚ &loss/total_loss‚18.685027571497045 _step‚635 epoch‚0 2loss/SRqqt2/assignment_loss‚0.7654325505970297 0loss/FBt2/assignment_loss‚0.0325112310441322 /loss/FRt1/detection_loss‚2.2957426999943165 1loss/SRqqt2/detection_loss‚2.0326377645739027 trainer/global_step‚7949 % _timestamp‚1.7858135999579136e+09 _runtime‚3345.147693251 0loss/FRt1/assignment_loss‚2.6208092494855952 2loss/SRqqt1/assignment_loss‚0.9637055080959552 /loss/FBt1/detection_loss‚2.5019248273223638 .loss/FRt2/assignment_loss‚2.78193399310112 0loss/FB©jÀt1/assignment_loss‚0.1993013911766866 .loss/FRt2/detection_loss‚2.593634119523423 1loss/SRqqt1/detection_loss‚0.9487513584723131 /loss/FBt2/detection_loss‚0.9486428781102101û Š† œË‚(:‘ î´ÅÓÒÈ„ƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚108.157' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚36.05233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ú#uÌ‚(:ê  î´ÅÓ±Ë΃ disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1289.out‚0 disk.nvme0n1.in‚ 10.54296875 disk.drbd1965.out‚0 disk.drbd1931.out‚0 disk.drbd1510.out‚0 network.recv‚507232* disk./.usagePercent‚32.792648487233315 disk.drbd1226.in‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 proc.cpu.threads‚29$ disk./.usageGB‚586.3658599853516 disk.drbd1938.out‚0$ disk.nvme0n1.out‚699.14794921875 disk.sda.in‚ 9.17578125 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 120.38671875 disk.sda.out‚ 120.38671875 disk.drbd1817.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 13.2509765625 disk.drbd1931.in‚0! disk.dm-0.out‚699.14794921875 disk.drbd2111.in‚0 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0$ proc.memory.rssMB‚11254.05859375 disk.sda2.in‚ 9.17578125 disk.drbd2041.in‚0% memory_percent‚41.241371631622314 cpu‚13.757233591054776 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd1226.out‚0 disk.drbd1828.out‚0 network.sent‚ 7.422266e+06 disk.drbd1817.out‚0 disk.drbd1550.out‚0 disk.drbd1673.in‚0 disk.dm-0.in‚ 10.54296875 disk.drbd1552.in‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.sda1.out‚0) proc.memory.availableMB‚ 38508.0546875 disk.drbd2078.in‚0 disk.rbd0.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd2041.out‚0y³:2Í- _wandbruntime‚3366  _runtime‚3366a‘»vÇÎÁ ' lr-AdamW/pg2‚0.00020615171969596348 trainer/global_step‚7999 $ _timestamp‚1.785813620851613e+09 _runtime‚ 3366.04363611 _step‚636 ' lr-AdamW/pg1‚0.00020615171969596348†ÌÌÏÆ ' lr-AdamW/pg2‚0.00020615171969596348 trainer/global_step‚7999 $ _timestamp‚1.785813620851613e+09 _runtime‚ 3366.04363611 _step‚636 ' lr-AdamW/pg1‚0.00020615171969596348ü†©@Ü2Ð- _wandbruntime‚3366  _runtime‚33668™öóÒ'ÑÌO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31498395 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144377607724544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7039065449472 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53148577 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚144892364285952 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 % _timestamp‚1.7858136208523066e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚137662758734848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚144377712193536 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚140947777117656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚140947674275172 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚84486625 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚3366.043728047 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7021751037700 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚84476969 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7038978019840 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53148577 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚133925940276832 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚84486625 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _step‚637 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7038978019840 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚144892202416640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚144377607724544TìB{×'ÒÑO N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53148579 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚137338646744064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31498397 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22918324 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7229580647424 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53148577 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚133925940276832 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚84486625 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _step‚637 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7038978019840 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚144892202416640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚144377607724544 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31498395 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144377607724544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7039065449472 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53148577 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚144892364285952 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 % _timestamp‚1.7858136208523066e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚137662758734848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚144377712193536 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚140947777117656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚140947674275172 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚84486625 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚3366.043728047 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7021751037700 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚84476969 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7038978019840 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚144395758583296 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 137652384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7039857639424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚133942663409792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚144910317578240 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚84487492 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚137679990755328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7022628510922 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 137652384 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚84497162 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚140965394763310 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22921171 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚137355796467712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚144910471051776 % _timestamp‚1.7858136212003584e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54423503 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚133942680449152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚7999 _runtime‚3366.390624368 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53155222 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53155220 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144395654107136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31502334 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚137355813507072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31502332 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7230351726592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚137355796467712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7039945076224 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚144395758583296 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7039945076224 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚84497162 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.sm4 º·Ìall_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚84487492 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7039857639424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53155220 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121Íú!á&ÕÛM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚144395758583296 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 137652384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7039857639424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚133942663409792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚144910317578240 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚84487492 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚137679990755328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7022628510922 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 137652384 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚84497162 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚140965394763310 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22921171 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚137355796467712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚144910471051776 % _timestamp‚1.7858136212003584e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54423503 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚133942680449152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚7999 _runtime‚3366.390624368 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53155222 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53155220 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144395654107136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31502334 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚137355813507072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31502332 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7230351726592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚137355796467712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7039945076224 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚144395758583296 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7039945076224 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚84497162 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚84487492 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7039857639424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53155220 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54423754 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22921420 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0þnGrI2Ö- _wandbruntime‚3366  _runtime‚3366'l×€ 0loss/SRqqt1/detection_loss‚1.023674383447055 4loss/FBt2/assignment_loss‚0.00016752847606814277 .loss/FRt2/detection_loss‚2.525367058338003 epoch‚0 _runtime‚3366.391336376 2loss/SRqqt1/assignment_loss‚1.0628276891675927 /loss/FRt1/detection_loss‚2.2919094791246013 1loss/SRqqt2/detection_loss‚1.8041136664619972 %loss/total_loss‚19.04261293996347 trainer/global_step‚7999 0loss/FRt1/assignment_loss‚2.7027070349039035 /loss/FRt2/assignment_loss‚2.933963267093009 .loss/FBt1/detection_loss‚2.517045554580788 /loss/FBt2/detection_loss‚1.0109708276577294 % _timestamp‚1.7858136212015588e+09 _step‚639 2loss/SRqqt2/assignment_loss‚1.1289257595618798 1loss/FBt1/assignment_loss‚0.04094069115083888:‹_î Ø… %loss/total_loss‚19.04261293996347 trainer/global_step‚7999 0loss/FRt1/assignment_loss‚2.7027070349039035 /loss/FRt2/assignment_loss‚2.933963267093009 .loss/FBt1/detection_loss‚2.517045554580788 /loss/FBt2/detection_loss‚1.0109708276577294 % _timestamp‚1.7858136212015588e+09 _step‚639 2loss/SRqqt2/assignment_loss‚1.1289257595618798 1loss/FBt1/assignment_loss‚0.04094069115083888 0loss/SRqqt1/detection_loss‚1.023674383447055 4loss/FBt2/assignment_loss‚0.00016752847606814277 .loss/FRt2/detection_loss‚2.525367058338003 epoch‚0 _runtime‚3366.391336376 2loss/SRqqt1/assignment_loss‚1.0628276891675927 /loss/FRt1/detection_loss‚2.2919094791246013 1loss/SRqqt2/detection_loss‚1.8041136664619972ÿô`ÜópÙ‚(:å  ý´ÅÓ€Öªƒ disk.drbd1289.out‚0 network.recv‚509401 disk.drbd2118.in‚0 disk.rbd0.out‚0 proc.cpu.threads‚29 disk.drbd1285.in‚0 disk.rbd1.in‚0 disk.drbd1817.in‚0 disk.sda2.in‚ 9.17578125 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 cpu‚13.756877163113083" disk.nvme0n1.out‚ 707.310546875 disk.drbd1226.in‚0 disk.sda.out‚ 122.91796875$ memory_percent‚41.24066233634949$ proc.memory.rssMB‚11254.05859375 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.nvme0n1.in‚ 10.54296875 disk.drbd1931.in‚0 disk.drbd2118.out‚0 disk.drbd1356.out‚0 disk.drbd1828.out‚0 disk.dm-0.in‚ 10.54296875 disk.dm-0.out‚ 707.310546875 disk.drbd1938.out‚0 disk.sda.in‚ 9.17578125 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 122.91796875 disk.rbd1.out‚ 13.283203125 disk.drbd1931.out‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 network.sent‚ 7.456943e+06 disk.sda1.in‚0* proc.memory.availableMB‚38508.51953125* disk./.usagePercent‚32.792647420544114$ disk./.usageGB‚586.3658409118652 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1552.out‚0 disk.drbd1584.in‚0 disk.drbd2078.in‚0 disk.drbd1356.in‚0 disk.drbd1965.out‚0 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd2041.out‚0* proc.memory.percent‚17.172330617904663¹ÁN³œÚ‚(:‘ ý´Åӡ穃 gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚123.978' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.32599999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ò’rz2Û- _wandbruntime‚3386  _runtime‚3386ÂW©ÏÈÜ ' lr-AdamW/pg2‚0.00020744032902022877 trainer/global_step‚8049 $ _timestamp‚1.785813641605335e+09 _runtime‚3386.795282214 _step‚640 ' lr-AdamW/pg1‚0.00020744032902022877*…cÍÝÇ _runtime‚3386.795282214 _step‚640 ' lr-AdamW/pg1‚0.00020744032902022877 ' lr-AdamW/pg2‚0.00020744032902022877 trainer/global_step‚8049 $ _timestamp‚1.785813641605335e+09€N Ù¥2Þ- _wandbruntime‚3386  _runtime‚3386gb4Ò'ßÌO YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚138196967855104 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚134762931853472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _runtime‚3386.799634812 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚138523114574848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 % _timestamp‚1.7858136416062713e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚145280046479360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53480729 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚145279942010368 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7083061584896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚85014627 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7082974155264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚138196984894464 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53480727 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚85004971 _step‚641 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31695247 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚85004971 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7274758953472 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7274619890688 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚141828674197164 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚138196984894464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚145279942010368 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚85014627 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚145797734465536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚141828571354680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54756371 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7065639501208 trainer/global_step‚8049 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 138495356 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚138196967855104 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53480729 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7065725304332 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54756633 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚145797898432000 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53480727 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚145279942010368 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7083061584896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚85014627 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7082974155264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚138196984894464 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53480727 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚85004971 _step‚641 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31695247 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚85004971 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7274758953472 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7274619890688 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚141828674197164 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚138196984894464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚145279942010368 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚85014627 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚145797734465536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚141828571354680 WJDeviceStatsMonitor.on_train_bœÊdcùatch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54756371 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7065639501208 trainer/global_step‚8049 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 138495356 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚138196967855104 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53480729 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7065725304332 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54756633½Î…~2á- _wandbruntime‚3387  _runtime‚3387¦ˆÎÛ&âÕM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53487372 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚141846291842818 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚141846189000222 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53487370 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 138502864 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7083853774848 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚3387.150658813 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚85025164 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7275526919680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7083853774848 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53487370 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚134779689065152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚134779672025792 trainer/global_step‚8049 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _step‚642 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚145298092869120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚85015494 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138512536 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7275655489536 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23064018 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚138540321691648 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54763200 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23064274 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 $ _timestamp‚1.785813641960379e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138512536 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚145815848611328 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚138540346595328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚145297988392960 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚85015494 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚138214151657472 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7066602777666 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚145298092869120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚138214134618112 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚145816002084864 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7066516974430 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚85025164 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856Ffà&ãÚM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚145298092869120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚138214134618112 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚145816002084864 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7066516974430 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚85025164 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53487372 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚141846291842818 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚141846189000222 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53487370 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 138502864 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7083853774848 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚3387.150658813 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚85025164 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7275526919680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7083853774848 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53487370 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚134779689065152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚134779672025792 trainer/global_step‚8049 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _step‚642 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚145298092869120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚85015494 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138512536 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7275655489536 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23064018 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚138540321691648 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54763200 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23064274 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 $ _timestamp‚1.785813641960379e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138512536 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚145815848611328 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚138540346595328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚145297988392960 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚85015494 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚138214151657472 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7066602777666‚!W.¡2ä- _wandbruntime‚3387  _runtime‚3387ÏÃZÑ å„ 1loss/SRqqt1/detection_loss‚0.9468460838735991 1loss/SRqqt2/detection_loss‚1.9399040795576115 2loss/SRqqt1/assignment_loss‚0.9986939304878331 1loss/FBt2/assignment_loss‚0.01826982663681668 /loss/FRt1/detection_loss‚2.2355522540338497 % _timestamp‚1.7858136419618695e+09 _runtime‚3387.151658852 .loss/FBt1/detection_loss‚1.280286491745048 &loss/total_loss‚17.301986464062026 _step‚643 /loss/FBt2/detection_loss‚1.1425603232769803 epoch‚0 0loss/FRt1/assignment_loss‚2.1283978737415152 0loss/FRt2/assignment_loss‚2.9038918250136905 2loss/SRqqt2/assignment_loss‚0.9876555093393947 4loss/FBt1/assignment_loss‚0.00016018839275804767 /loss/FRt2/detection_loss‚2.7197680779629283 trainer/global_step‚8049dÕíëæ‰ .loss/FBt1/detection_loss‚1.280286491745048 &loss/total_loss‚17.301986464062026 _step‚643 /loss/FBt2/detection_loss‚1.1425603232769803 epoch‚0 0loss/FRt1/assignment_loss‚2.1283978737415152 0loss/FRt2/assignment_loss‚2.9038918250136905 2loss/SRqqt2/assignment_loss‚0.9876555093393947 4loss/FBt1/assignment_loss‚0.00016018839275804767 /loss/FRt2/detection_loss‚2.7197680779629283 trainer/global_step‚8049 1loss/SRqqt1/detection_loss‚0.9468460838735991 1loss/SRqqt2/detection_loss‚1.9399040795576115 2loss/SRqqt1/assignment_loss‚0.9986939304878331 1loss/FBt2/assignment_loss‚0.01826982663681668 /loss/FRt1/detection_loss‚2.2355522540338497 % _timestamp‚1.7858136419618695e+09 _runtime‚3387.151658852ƒz)ôlœç‚(:‘ ŒµÅÓý”„ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚129.697' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.23233333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ç,n‰hè‚(:Ý  ŒµÅÓïÛ˃ disk.drbd1828.in‚0 disk.dm-0.in‚ 10.54296875 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.sda.in‚ 9.17578125 disk.sda2.in‚ 9.17578125 disk.drbd1828.out‚0 disk.drbd1938.out‚0 disk.drbd1584.in‚0 cpu‚13.756555534260547 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd1931.in‚0* proc.memory.availableMB‚38508.23828125* disk./.usagePercent‚32.792643153787296$ disk./.usageGB‚586.3657646179199 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.sda.out‚124.1875 disk.drbd1356.out‚0$ memory_percent‚41.24109148979187 proc.cpu.threads‚29 disk.drbd2078.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.drbd1226.out‚0 disk.drbd1510.out‚0 disk.drbd1356.in‚0 disk.drbd1510.in‚0 disk.drbd1552.in‚0 network.recv‚511662 disk.drbd1965.in‚0 disk.drbd1817.out‚0" disk.nvme0n1.out‚ 708.435546875 disk.rbd0.in‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.sda2.out‚124.1875 disk.rbd1.out‚ 13.283203125 disk.nvme0n1.in‚ 10.54296875 disk.drbd2041.out‚0 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.sda1.out‚0 disk.drbd2118.out‚0$ proc.memory.rssMB‚11254.05859375 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.dm-0.out‚ 708.435546875* proc.memory.percent‚17.172330617904663 disk.drbd1226.in‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 network.sent‚ 7.502248e+06²j´é‚(:’ ›µÅӸŃ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚107.664' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚35.888000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡ùÄÒsê‚(:è  ›µÅÓˬ΃ disk.drbd1938.out‚0 network.sent‚ 7.547474e+06 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.sda2.in‚ 9.17578125 disk.drbd1965.in‚0$ proc.memory.rssMB‚11254.05859375 cpu‚13.756094496577138 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.drbd1938.in‚0$ disk.nvme0n1.out‚710.72607421875) disk./.usagePercent‚32.79264208709809 disk.drbd1817.out‚0 disk.drbd1584.in‚0 disk.rbd0.out‚0$ memory_percent‚41.24071002006531 network.recv‚514011 disk.drbd1285.out‚0 disk.drbd1356.in‚0 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.drbd1510.in‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.rbd0.in‚0 disk.drbd2078.in‚0 disk.nvme0n1.in‚ 10.54296875 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd1828.in‚0 disk.sda1.in‚0 disk.sda2.out‚ 124.6015625$ disk./.usageGB‚586.3657455444336 disk.drbd1285.in‚0 disk.drbd1289.out‚0 disk.drbd2118.in‚0 disk.rbd1.in‚0 disk.dm-0.in‚ 10.54296875! disk.dm-0.out‚710.72607421875 disk.rbd1.out‚13.34619140625 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.sda.in‚ 9.17578125 disk.sda.out‚ 124.6015625 disk.drbd1226.out‚0 disk.drbd1356.out‚0* proc.memory.availableMB‚38508.48828125 disk.drbd1828.out‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.drbd1965.out‚0* proc.memory.percent‚17.172330617904663 proc.cpu.threads‚29 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0P—2ë- _wandbruntime‚3407  _runtime‚3407„½s•Éìà _runtime‚3407.436431567 _step‚644 ' lr-AdamW/pg1‚0.00020872893834449408 ' lr-AdamW/pg2‚0.00020872893834449408 trainer/global_step‚8099 % _timestamp‚1.7858136622465043e+09w½¾ÎíÈ _runtime‚3407.436431567 _step‚644 ' lr-AdamW/pg1‚0.00020872893834449408 ' lr-AdamW/pg2‚0.00020872893834449408 trainer/global_step‚8099 % _timestamp‚1.7858136622465043e+09„ãɵH2î- _wandbruntime‚3407  _runtime‚3407E~JPïÌO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚85532973 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚139055323044864 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚139055306005504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚273 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚146703225591808 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55096380 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚146703389558272 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚146182380765184 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 139355508 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚146182276296192 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7319755176960 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7319894239744 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 139345850 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7126970290688 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.Ÿ¿²‚large_pool.freed‚139383470414848 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31892095 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚8099 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚146182380765184 % _timestamp‚1.7858136622474296e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚146182276296192 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53812879 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7127057720320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53812879 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚85542629 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7127057720320 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53812877 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚85542629 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚139383495318528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53812877 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚139055323044864 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7126970290688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚139055306005504 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23204283 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚135599940469472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3407.440628428 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚142709571276672 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚85532973 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55096105 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23204010 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚139055306005504 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23204283 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚135599940469472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3407.440628428 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚142709571276672 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚85532973 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55096105 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23204010 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚146182276296192 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7319755176960 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7319894239744 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 139345850 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7126970290688 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚139383470414848 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31892095 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 trainer/global_step‚8099 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚146182380765184 % _timestamp‚1.7858136622474296e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚146182276296192 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53812879 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7127057720320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936…{¹‡k2ñ- _wandbruntime‚3407  _runtime‚3407«AŽðÜ&òÖM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55102915 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚142727086079730 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7110491241174 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 % _timestamp‚1.7858136626042066e+09 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 166056448 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53819520 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚8099 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚146721331021312 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7127937347072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7127849910272 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7320653489664 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139372688 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31896034 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53819520 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚85543496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7127937347072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚146721497077760 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7320794642432 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7110405437938 _runtime‚3407.794474275 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚139072489807872 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7127849910272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚146200322678784 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚85553166 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _step‚646 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139363016 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚142727188922326 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53819522 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚85553166 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚139072472768512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚139400702435328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚146200427154944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚85543496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚139400677531648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚135616680641792 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚07颼vóÛM E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚85553166 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _step‚646 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139363016 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚142727188922326 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53819522 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚85553166 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚139072472768512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚139400702435328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚146200427154944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚85543496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚139400677531648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚135616680641792 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55102915 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚142727086079730 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7110491241174 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 % _timestamp‚1.7858136626042066e+09 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 166056448 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53819520 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚8099 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚146721331021312 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7127937347072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7127849910272 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7320653489664 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139372688 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31896034 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53819520 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚85543496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7127937347072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚146721497077760 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7320794642432 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7110405437938 _runtime‚3407.794474275 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚139072489807872 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7127849910272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚146200322678784 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200†ç ,´2ô- _wandbruntime‚3407  _runtime‚3407^g†Ðõý 0loss/FRt1/assignment_loss‚2.2785914572079977 2loss/SRqqt1/assignment_loss‚0.9468042716081884 epoch‚0 trainer/global_step‚8099 _step‚647 1loss/SRqqt2/detection_loss‚2.2599009726292048 /loss/FBt2/detection_loss‚0.7038864330388606 $ _timestamp‚1.785813662605404e+09 /loss/FRt2/assignment_loss‚2.915414997736613 2loss/SRqqt2/assignment_loss‚0.8778112981563959 1loss/FBt1/assignment_loss‚0.14911835989254782 2loss/FBt2/assignment_loss‚0.010682075385821024 .loss/FRt1/detection_loss‚2.435656863848368 1loss/SRqqt1/detection_loss‚1.0625500213354826 .loss/FBt1/detection_loss‚1.316947231945746 &loss/total_loss‚17.729953008680027 .loss/FRt2/detection_loss‚2.772589025894801 _runtime‚ 3407.79518918·Ü$ö‚ 1loss/SRqqt2/detection_loss‚2.2599009726292048 /loss/FBt2/detection_loss‚0.7038864330388606 $ _timestamp‚1.785813662605404e+09 /loss/FRt2/assignment_loss‚2.915414997736613 2loss/SRqqt2/assignment_loss‚0.8778112981563959 1loss/FBt1/assignment_loss‚0.14911835989254782 2loss/FBt2/assignment_loss‚0.010682075385821024 .loss/FRt1/detection_loss‚2.435656863848368 1loss/SRqqt1/detection_loss‚1.0625500213354826 .loss/FBt1/detection_loss‚1.316947231945746 &loss/total_loss‚17.729953008680027 .loss/FRt2/detection_loss‚2.772589025894801 _runtime‚ 3407.79518918 0loss/FRt1/assignment_loss‚2.2785914572079977 2loss/SRqqt1/assignment_loss‚0.9468042716081884 epoch‚0 trainer/global_step‚8099 _step‚647‡i@Ým÷‚(:„ ªµÅӼ𡃠gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.081' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.027 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0^aµctø‚(:é  ªµÅÓ±ŠØƒ disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.rbd0.out‚0 cpu‚13.755780515331303 proc.cpu.threads‚29$ disk./.usageGB‚586.3662605285645 disk.drbd1584.out‚0 disk.drbd1965.in‚0 disk.drbd1552.in‚0 disk.rbd1.out‚13.34619140625 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.drbd1828.in‚0 disk.nvme0n1.in‚ 10.54296875$ disk.nvme0n1.out‚716.39306640625 disk.drbd1584.in‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1673.out‚0 network.recv‚515087 disk.drbd1510.out‚0 disk.drbd1938.out‚0 disk.sda.in‚ 9.17578125 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.drbd1285.out‚0! disk.dm-0.out‚716.39306640625 disk.drbd1510.in‚0 disk.drbd2118.out‚0 disk.drbd2078.in‚0 network.sent‚ 7.550578e+06 disk.drbd1817.in‚0 disk.sda2.in‚ 9.17578125 disk.drbd1965.out‚0 disk.rbd0.in‚0* proc.memory.percent‚17.172330617904663 disk.dm-0.in‚ 10.54296875 disk.drbd1289.out‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd1550.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd2041.in‚0 disk.drbd1817.out‚0( proc.memory.availableMB‚ 38508.171875$ proc.memory.rssMB‚11254.05859375 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.drbd1828.out‚0$ memory_percent‚41.24119281768799 disk.drbd1285.in‚0* disk./.usagePercent‚32.792670887706606 disk.sda2.out‚ 124.98046875 disk.drbd1356.out‚0 disk.sda.out‚ 124.98046875*Ú²ð2ù- _wandbruntime‚3428  _runtime‚3428E/iÆúÀ _step‚648 & lr-AdamW/pg1‚0.0002100175476687594 & lr-AdamW/pg2‚0.0002100175476687594 trainer/global_step‚8149 % _timestamp‚1.7858136833531451e+09 _runtime‚ 3428.543072956W@ËûÅ _step‚648 & lr-AdamW/pg1‚0.0002100175476687594 & lr-AdamW/pg2‚0.0002100175476687594 trainer/global_step‚8149 % _timestamp‚1.7858136833531451e+09 _runtime‚ 3428.54307295ˆ¶C/2ü- _wandbruntime‚3428  _runtime‚3428ÇÍ´ÔÑ'ýËO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 trainer/global_step‚8149 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54145029 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23345682 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚143590468356180 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7171053855744 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚86060975 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7153416428224 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147084610582016 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32088945 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚139913661195264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7170966426112 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7153502231348 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54145027 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54145027 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7365032249344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚140243826254848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140206002 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚140243851158528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 140215660 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚86070631 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚147608723635712 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚147084610582016 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚147084715051008 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55434627 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚136436966124832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23345937 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7364897380864 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚86070631 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55434884 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚136436949085472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚649 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7171053855744 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32088947 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚143590365513696 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7858136833540766e+09 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54145029 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚86060975 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚147608883407872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484êmSœÖ'þÐO NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7858136833540766e+09 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54145029 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚86060975 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚147608883407872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 trainer/global_step‚8149 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54145029 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23345682 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚143590468356180 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7171053855744 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚86060975 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7153416428224 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147084610582016 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32088945 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚139913661195264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7170966426112 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7153502231348 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54145027 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54145027 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7365032249344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚140243826254848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140206002 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚140243851158528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 140215660 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚86070631 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚147608723635712 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚147084610582016 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚147084715051008 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55434627 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚136436966124832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23345937 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7364897380864 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚86070631 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55434884 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚136436949085472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚649 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7171053855744 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32088947 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚143590365513696 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640‰Â4€e2ÿ- _wandbruntime‚3428  _runtime‚3428^¨¤B%€ÕM =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7858136837027287e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7171933482496 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7365935562240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23348775 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚8149 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚86081168 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚147102761440768 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚147102656964608 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140232840 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚86081168 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23348529 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7154379704682 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚86071498 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚139930827958272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚147102761440768 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140223168 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7154293901446 _runtime‚ 3428.89300234 _step‚650 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54151670 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32092884 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚143607983159238 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7365800700928 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚136453706297152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55441411 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.séO8¶mall_pool.freed‚ 7171846045696 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7171846045696 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚86071498 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54151670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚140261058275328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140223168 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140232840 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7171933482496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚147626993837568 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32092882 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚54151672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55441659 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚143608086001834 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640E7Ëjà&ÚM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55441411 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7171846045696 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7171846045696 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚86071498 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54151670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚140261058275328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140223168 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140232840 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7171933482496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚147626993837568 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32092882 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚54151672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55441659 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚143608086001834 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7858136837027287e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7171933482496 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7365935562240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23348775 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚8149 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚86081168 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚147102761440768 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚147102656964608 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140232840 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚86081168 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23348529 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7154379704682 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚86071498 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚139930827958272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚147102761440768 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140223168 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7154293901446 _runtime‚ 3428.89300234 _step‚650 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54151670 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32092884 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚143607983159238 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7365800700928 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚136453706297152Šm5 [2‚- _wandbruntime‚3428  _runtime‚3428Ä' ƒ… 2loss/SRqqt1/assignment_loss‚0.9527120619963902 /loss/FBt2/detection_loss‚0.9785640598856844 _runtime‚3428.893724193 0loss/FRt2/assignment_loss‚2.3059208806823284 2loss/SRqqt2/assignment_loss‚0.9829951482938557 4loss/FBt2/assignment_loss‚2.8418023999776665e-05 epoch‚0 _step‚651 0loss/FRt1/assignment_loss‚2.3998964098562676 4loss/FBt1/assignment_loss‚1.0584511652922401e-05 /loss/FRt1/detection_loss‚2.4475249528256526 0loss/SRqqt1/detection_loss‚1.016751446815619 1loss/SRqqt2/detection_loss‚2.3111569702443555 %loss/total_loss‚17.18456921439271 trainer/global_step‚8149 .loss/FRt2/detection_loss‚2.531409364237505 /loss/FBt1/detection_loss‚1.2575989170193982 % _timestamp‚1.7858136837039213e+09܈f7„Š 2loss/SRqqt1/assignment_loss‚0.9527120619963902 /loss/FBt2/detection_loss‚0.9785640598856844 _runtime‚3428.893724193 0loss/FRt2/assignment_loss‚2.3059208806823284 2loss/SRqqt2/assignment_loss‚0.9829951482938557 4loss/FBt2/assignment_loss‚2.8418023999776665e-05 epoch‚0 _step‚651 0loss/FRt1/assignment_loss‚2.3998964098562676 4loss/FBt1/assignment_loss‚1.0584511652922401e-05 /loss/FRt1/detection_loss‚2.4475249528256526 0loss/SRqqt1/detection_loss‚1.016751446815619 1loss/SRqqt2/detection_loss‚2.3111569702443555 %loss/total_loss‚17.18456921439271 trainer/global_step‚8149 .loss/FRt2/detection_loss‚2.531409364237505 /loss/FBt1/detection_loss‚1.2575989170193982 % _timestamp‚1.7858136837039213e+09‹Þ±fS›…‚(: ¹µÅÓ€±žƒ gpu.0.gpu‚30.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.778' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.25933333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0|л3r†‚(:ç  ¹µÅÓþÞრdisk.drbd1285.in‚0 disk.sda.out‚ 127.5859375 disk.drbd1673.in‚0* proc.memory.availableMB‚38508.11328125 disk.drbd2078.out‚0 disk.drbd1552.in‚0 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.drbd1938.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2118.out‚0 disk.drbd1931.in‚0 disk.drbd2111.in‚0 disk.drbd1289.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd2078.in‚0 disk.drbd1817.in‚0 disk.rbd1.out‚ 13.4091796875 disk.sda.in‚ 9.17578125 disk.dm-0.in‚ 10.54296875 disk.sda1.out‚0 disk.drbd1584.out‚0 disk.rbd0.out‚0 cpu‚13.755540693215593 proc.cpu.threads‚29 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚ 10.54296875 disk.drbd1356.out‚0 disk.drbd2111.out‚0 disk.drbd1510.out‚0 disk.drbd1817.out‚0 disk.drbd1931.out‚0$ memory_percent‚41.24128222465515! disk.dm-0.out‚721.70849609375 network.sent‚ 7.595619e+06) disk./.usagePercent‚32.79310311017217 disk.drbd1550.out‚0 disk.sda2.out‚ 127.5859375 disk.drbd1584.in‚0 network.recv‚517407$ disk./.usageGB‚586.3739891052246 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd1510.in‚0 disk.sda2.in‚ 9.17578125 disk.drbd1285.out‚0 disk.drbd1356.in‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0$ disk.nvme0n1.out‚721.70849609375 disk.drbd2041.in‚0 disk.drbd2041.out‚0a 62‡- _wandbruntime‚3449  _runtime‚3449ÚÞ"ƈÀ & lr-AdamW/pg1‚0.0002113061569930247 & lr-AdamW/pg2‚0.0002113061569930247 trainer/global_step‚8199 $ _timestamp‚1.785813703959196e+09 _runtime‚3449.150627896 _step‚652oÉ…yˉŠ$ _timestamp‚1.785813703959196e+09 _runtime‚3449.150627896 _step‚652 & lr-AdamW/pg1‚0.0002113061569930247 & lr-AdamW/pg2‚0.0002113061569930247 trainer/global_step‚8199Œãñ@H2Š- _wandbruntime‚3449  _runtime‚3449p[ÆÑ'‹ËO @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚141104206998528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7215049991168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚141104182094848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚144471262593204 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147986944867840 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55774193 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚3449.150731637 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚140771982306304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23488141 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚144471365435688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚140771982306304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23488396 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚86588977 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7197304891732 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32285797 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 $ _timestamp‚1.785813703959864e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7197390694856eQ“¾ï ŒÐO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 141066154 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚140771999345664 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚148514266719232 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23488141 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚144471365435688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚140771982306304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23488396 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚86588977 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7197304891732 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32285797 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 $ _timestamp‚1.785813703959864e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7197390694856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚141104206998528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7215049991168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚141104182094848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚144471262593204 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147986944867840 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55774193 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚3449.150731637 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚140771982306304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚140789166108672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚86599500 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚141121389211648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚54483822 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141083320 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚8199 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚148005095726592 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23491249 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚3449.501474685 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54483820 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55780737 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55780983 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚144488983081342 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32289734 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148005095726592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7215929617920 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 141083320 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141092992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚137290697873792Isuèà&ÚM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚86599500 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚141121389211648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚54483822 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141083320 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚8199 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚148005095726592 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23491249 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚3449.501474685 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54483820 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55780737 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55780983 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚144488983081342 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32289734 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148005095726592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7215929617920 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 141083320 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141092992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚137290697873792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚140789166108672ŽçÙ´2- _wandbruntime‚3449  _runtime‚3449–Œ×0‘þ trainer/global_step‚8199 /loss/FRt1/assignment_loss‚2.614709782984949 1loss/SRqqt2/assignment_loss‚1.137390105422431 2loss/FBt1/assignment_loss‚0.042961602959621814 1loss/FBt2/assignment_loss‚0.01287198718165842 /loss/FBt1/detection_loss‚0.6478801608424295 % _timestamp‚1.7858137043123777e+09 1loss/SRqqt1/detection_loss‚0.9157570814842294 _step‚655 /loss/FBt2/detection_loss‚1.3071071323938668 %loss/total_loss‚17.86966518047775 epoch‚0 _runtime‚ 3449.50215432 0loss/FRt2/assignment_loss‚3.2540469272637074 2loss/SRqqt1/assignment_loss‚1.0000001906706342 .loss/FRt1/detection_loss‚2.364209734303977 /loss/FRt2/detection_loss‚2.6585916575090387 1loss/SRqqt2/detection_loss‚1.9141388174612075Å’UË ’ƒ 2loss/SRqqt1/assignment_loss‚1.0000001906706342 .loss/FRt1/detection_loss‚2.364209734303977 /loss/FRt2/detection_loss‚2.6585916575090387 1loss/SRqqt2/detection_loss‚1.9141388174612075 trainer/global_step‚8199 /loss/FRt1/assignment_loss‚2.614709782984949 1loss/SRqqt2/assignment_loss‚1.137390105422431 2loss/FBt1/assignment_loss‚0.042961602959621814 1loss/FBt2/assignment_loss‚0.01287198718165842 /loss/FBt1/detection_loss‚0.6478801608424295 % _timestamp‚1.7858137043123777e+09 1loss/SRqqt1/detection_loss‚0.9157570814842294 _step‚655 /loss/FBt2/detection_loss‚1.3071071323938668 %loss/total_loss‚17.86966518047775 epoch‚0 _runtime‚ 3449.50215432 0loss/FRt2/assignment_loss‚3.2540469272637074 Ë\ùœ“‚(:‘ ȵÅÓÙÂ…ƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.829' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚32.276333333333326 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0DŒNs”‚(:è  ȵÅÓÃÛσ disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0$ disk./.usageGB‚586.7099647521973 disk.rbd0.out‚0 disk.drbd2111.in‚0 disk.drbd1289.in‚0 disk.drbd1673.in‚0% memory_percent‚41.241633892059326$ proc.memory.rssMB‚11254.05859375* proc.memory.percent‚17.172330617904663 disk.sda.in‚ 9.17578125 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.drbd1356.in‚0 disk.drbd1965.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 9.17578125 disk.drbd1931.in‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0) proc.memory.availableMB‚ 38507.8828125 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd2041.out‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.dm-0.out‚830.0048828125 disk.drbd2111.out‚0 disk.dm-0.in‚ 10.73828125* disk./.usagePercent‚32.811892627167104 disk.drbd1550.out‚0 network.recv‚519952 disk.drbd2041.in‚0 disk.sda.out‚ 129.63671875 disk.drbd1285.in‚0 disk.drbd2078.in‚0 disk.drbd1289.out‚0 disk.nvme0n1.in‚ 10.73828125 disk.rbd1.out‚ 13.4091796875 proc.cpu.threads‚29 disk.drbd1285.out‚0# disk.nvme0n1.out‚830.0048828125 disk.drbd1584.out‚0 disk.drbd1828.in‚0 disk.drbd1673.out‚0 disk.drbd1226.in‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 network.sent‚ 7.640726e+06 disk.drbd1550.in‚0 disk.drbd1510.out‚0 disk.drbd2118.in‚0 disk.sda2.out‚ 129.63671875 cpu‚13.755162375820236 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1356.out‚0)gZk•‚(:’ ×µÅÓ²«ƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚126.882' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚42.294000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ç“–Áô–‚(:ä  ×µÅÓ¾»áƒ disk.drbd1938.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 proc.cpu.threads‚29 network.recv‚522203 disk.sda1.in‚0 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.drbd1828.out‚0 disk.drbd2111.out‚0* proc.memory.availableMB‚38508.13671875* proc.memory.percent‚17.172330617904663$ disk./.usageGB‚586.7215614318848 disk.drbd1552.out‚0 disk.drbd1584.out‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0# disk.nvme0n1.out‚842.3720703125 disk.sda.out‚ 131.55859375 disk.drbd1828.in‚0 disk.dm-0.out‚842.3720703125% memory_percent‚41.241246461868286$ proc.memory.rssMB‚11254.05859375 disk.sda2.in‚ 9.17578125 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1510.out‚0) disk./.usagePercent‚32.81254117420328 disk.drbd1673.out‚0 disk.drbd1931.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 cpu‚13.7545724092039 disk.drbd1226.in‚0 disk.drbd1285.out‚0 disk.rbd1.in‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd1510.in‚0 disk.sda2.out‚ 131.55859375 disk.rbd1.out‚ 13.5576171875 network.sent‚ 7.685779e+06 disk.drbd1817.in‚0 disk.drbd2078.in‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.drbd1938.out‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.drbd2118.in‚0 disk.nvme0n1.in‚ 28.8046875 disk.sda.in‚ 9.17578125 disk.rbd0.in‚0½C¯•{ disk.rbd0.out‚0 disk.dm-0.in‚ 28.8046875 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0l]à2—- _wandbruntime‚3470  _runtime‚3470¨6zȘ $ _timestamp‚1.785813724815081e+09 _runtime‚3470.005016624 _step‚656 ' lr-AdamW/pg1‚0.00021259476631728998 ' lr-AdamW/pg2‚0.00021259476631728998 trainer/global_step‚82491^!ꪂ trainer/global_step‚8249 $ _timestamp‚1.785813724815081e+09 _runtime‚3470.005016624 _step‚656 ' lr-AdamW/pg1‚0.00021259476631728998 ' lr-AdamW/pg2‚0.000212594766317289982š- _wandbruntime‚3470  _runtime‚3470Ò‰iÑ'›ËO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚148889279153664 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚141964562838528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56113555 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56113802 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚657 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7258958696960 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54809329 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7455247021056 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32482645 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23630910 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚141630320456704 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚149419946825216 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138110983356832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚149419784955904 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54809327 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚148889383622656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7259046126592 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 141935964 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7455383986688 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54809329 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚87126635 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858137248160095e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚141630337496064 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32482647 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚145352159672712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7259046126592 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚141964537934848 _runtime‚ 3470.00966296 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚141630337496064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121½ÚÝÖ'œÐO ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚148889279153664 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚141964562838528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56113555 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56113802 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚657 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7258958696960 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54809329 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7455247021056 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32482645 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23630910 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚141630320456704 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚149419946825216 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138110983356832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚149419784955904 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54809327 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚148889383622656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7259046126592 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 141935964 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7455383986688 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54809329 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚87126635 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 % _timestamp‚1.7858137248160095e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚141630337496064 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32482647 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚145352159672712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7259046126592 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚141964537934848 _runtime‚ 3470.00966296 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚141630337496064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0‘hÆ2- _wandbruntime‚3470  _runtime‚3470ûù‡Ü&žÖM EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56120290 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚87127502 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23633960 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7259925753344 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚145369880160850 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56120544 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚149437896506880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚148907430012416 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 _runtime‚3470.359318428 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32486584 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7456280025088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚141647504259072 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚87137172 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7259838316544 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚87137172 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚148907325536256 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7456151455232 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7242156631698 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54815972 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 trainer/global_step‚8249 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141943472 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148907430012416 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚141981745051648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚138127723529152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚138127706489792 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 141943472 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141953144 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚149438049980416 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23633708 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _step‚658 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚141647487219712 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141953144 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚149438049980416 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23633708 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _step‚658 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚141647487219712 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56120290 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚87127502 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23633960 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7259925753344 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚145369880160850 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56120544 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚149437896506880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚148907430012416 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 _runtime‚3470.359318428 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32486584 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7456280025088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚141647504259072 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚87137172 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7259838316544 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚87137172 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚148907325536256 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7456151455232 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7242156631698 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54815972 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 trainer/global_step‚8249 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141943472 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148907430012416 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚141981745051648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚138127723529152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚138127706489792 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236’Qþ;2 - _wandbruntime‚3470  _runtime‚3470¹¿1 ¡„ /loss/FBt1/detection_loss‚1.0832460189915516 % _timestamp‚1.7858137251702495e+09 _runtime‚3470.360049125 3loss/FBt2/assignment_loss‚0.0015271333632833094 .loss/FRt1/detection_loss‚2.438427986204624 0loss/FRt2/assignment_loss‚2.7828872694688687 1loss/SRqqt1/assignment_loss‚0.941564501752001 1loss/SRqqt2/assignment_loss‚0.663675294554931 /loss/FRt2/detection_loss‚2.8137936745496357 1loss/SRqqt1/detection_loss‚1.0203190482574405 1loss/SRqqt2/detection_loss‚1.5275230892597045 /loss/FBt2/detection_loss‚0.7043652882909074 &loss/total_loss‚16.341210542487552 epoch‚0 trainer/global_step‚8249 _step‚659 0loss/FRt1/assignment_loss‚2.3635909125208854 4loss/FBt1/assignment_loss‚0.00029032527372020343ŽE‘¢‰ 1loss/SRqqt1/assignment_loss‚0.941564501752001 1loss/SRqqt2/assignment_loss‚0.663675294554931 /loss/FRt2/detection_loss‚2.8137936745496357 1loss/SRqqt1/detection_loss‚1.0203190482574405 1loss/SRqqt2/detection_loss‚1.5275230892597045 /loss/FBt2/detection_loss‚0.7043652882909074 &loss/total_loss‚16.341210542487552 epoch‚0 trainer/global_step‚8249 _step‚659 0loss/FRt1/assignment_loss‚2.3635909125208854 4loss/FBt1/assignment_loss‚0.00029032527372020343 /loss/FBt1/detection_loss‚1.0832460189915516 % _timestamp‚1.7858137251702495e+09 _runtime‚3470.360049125 3loss/FBt2/assignment_loss‚0.0015271333632833094 .loss/FRt1/detection_loss‚2.438427986204624 0loss/FRt2/assignment_loss‚2.7828872694688687“z5Û£‚(: æµÅÓÖµú‚ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.469' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.15633333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€¹ Ko¤‚(:ä  æµÅÓµëÀƒ disk.drbd1931.out‚0$ memory_percent‚41.24097228050232 cpu‚13.754308165629975 disk.rbd0.out‚0 disk.drbd1550.in‚0 disk.drbd1356.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1550.out‚0 disk.sda1.out‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd1356.out‚0 disk.dm-0.out‚984.8095703125 disk.drbd1552.out‚0# disk.nvme0n1.out‚984.8095703125* proc.memory.availableMB‚38508.31640625 network.recv‚523333( disk./.usagePercent‚32.8058814068255 disk.drbd1289.in‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚ 28.8046875 disk.drbd2041.out‚0 network.sent‚ 7.688967e+06 disk.drbd1965.out‚0 disk.drbd1828.in‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 9.17578125 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.sda.out‚ 132.03515625 proc.cpu.threads‚29 disk.drbd1285.in‚0 disk.dm-0.in‚ 28.8046875 disk.drbd1817.in‚0 disk.drbd1226.out‚0$ disk./.usageGB‚586.6024780273438 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.drbd1584.out‚0 disk.sda1.in‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.drbd1931.in‚0 disk.sda.in‚ 9.17578125 disk.drbd2111.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 13.5576171875 disk.drbd1584.in‚0 disk.drbd2041.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd1552.in‚0 disk.sda2.out‚ 132.03515625 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1938.out‚0óFÜ…2¥- _wandbruntime‚3490  _runtime‚3490zg.êɦà _step‚660 ' lr-AdamW/pg1‚0.00021388337564155532 ' lr-AdamW/pg2‚0.00021388337564155532 trainer/global_step‚8299 % _timestamp‚1.7858137456382422e+09 _runtime‚3490.828200012åÌZÎ§È ' lr-AdamW/pg2‚0.00021388337564155532 trainer/global_step‚8299 % _timestamp‚1.7858137456382422e+09 _runtime‚3490.828200012 _step‚660 ' lr-AdamW/pg1‚0.00021388337564155532”q²û2¨- _wandbruntime‚3490  _runtime‚3490£k`+Ò'©ÌO O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚142824893774848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7285167621872 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚142488675646464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚142488658607104 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23773767 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 % _timestamp‚1.7858137456392384e+09 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚150325281183232 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7303042262016 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚149791717908480 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55141477 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56453508 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚87644981 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 142796116 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55141479 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚149791613439488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚142488675646464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚661 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7500387408384 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138947991972832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚150325436761088 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚146233056752220 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 trainer/global_step‚8299 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7302954832384 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 142786458 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32679495 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚87644981 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7303042262016 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚142824918678528 _runtime‚3490.832674009 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55141477 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7500518082560 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚87654637 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23774011 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚87654637 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7302954832384 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32679497 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚149791717908480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55141479 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7285081818748 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56453508 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚87644981 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 142796116 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55141479 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚149791613439488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚142488675646464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚661 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7500387408384 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138947991972832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚150325436761088 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚146233056752220 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 trainer/global_step‚8299 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7302954832384 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 142786458 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32679495 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚87644981 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7303042262016 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚142824918678528 _runtime‚3490.832674009 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55141477 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7500518082560 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚87654637 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23774011 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚87654637 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7302954832384 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32679497 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚149791717908480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55141479 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7285081818748 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32683434 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚87665174 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7303921888768 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7303834451968 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚149809764298240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 142813296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚142505842409472 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56460073 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 trainer/global_step‚8299 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7285959291970 _step‚662 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7501290291200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146250674397762 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚87655504h¦Áá&­ÛM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55148122 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55148120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚253 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚142505825370112 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55148120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23776892 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7501425152512 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚87665174 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23776641 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚138964732145152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32683434 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚87665174 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7303921888768 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7303834451968 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚149809764298240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 142813296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚142505842409472 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56460073 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 trainer/global_step‚8299 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7285959291970 _step‚662 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7501290291200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146250674397762 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚87655504 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚150343550947840 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚142842125795328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 142803624 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚87655504 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7303834451968 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32683432 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚150343391182848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55473627 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚143346996757504 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7347038397440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56793621 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23917276 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 143646610 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚143347013796864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32876347 trainer/global_step‚8349 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23917547 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 _step‚665 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55473629 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚273 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7346950967808 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚143685274518528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚88182639 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚150693947725312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7346950967808 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 # _timestamp‚1.78581376666744e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55473629 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚88172983 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚147114056674212 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚147113953831728 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7328970282256 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚150693947725312 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _runtime‚3511.858718103 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32876345 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7329056085380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 143656268 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 30219960328Vì&Õ'ºÏO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32876345 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7329056085380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 143656268 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚271 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚150694052194304 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚139784983549472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7347038397440 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55473627 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚143346996757504 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7347038397440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56793621 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23917276 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 143646610 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚143347013796864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32876347 trainer/global_step‚8349 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23917547 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 _step‚665 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55473629 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚273 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7346950967808 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚143685274518528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚88182639 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚150693947725312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7346950967808 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 # _timestamp‚1.78581376666744e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55473629 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚88172983 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚147114056674212 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚147113953831728 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7328970282256 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚150693947725312 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _runtime‚3511.858718103™Qƒ·ö2»- _wandbruntime‚3512  _runtime‚3512ZÛɦÜ&¼ÖM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55480270 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56800422 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚143702481635328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚3512.208215506 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚88193176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7546510149632 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 143663776 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 143673448 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55480272 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56800658 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚88193176 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚143702456731648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚147131674319866 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7347918024192 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚88183506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7546634525184 % _timestamp‚1.7858137670179384e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚88183506 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚143364163520512 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0Œ–цí!½ÛM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7546510149632 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 143663776 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 143673448 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55480272 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56800658 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚88193176 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚143702456731648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚147131674319866 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7347918024192 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚88183506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7546634525184 % _timestamp‚1.7858137670179384e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚88183506 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚143364163520512 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55480270 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56800422 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚143702481635328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚3512.208215506 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚88193176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0šÍ)2¾- _wandbruntime‚3512  _runtime‚3512•Ä™ú ¿„ 3loss/FBt1/assignment_loss‚3.585129127928025e-05 1loss/SRqqt2/detection_loss‚1.8843254211048286 /loss/FBt2/detection_loss‚1.6070876019696394 _step‚667 0loss/FRt1/assignment_loss‚2.2601176494801485 0loss/FRt2/assignment_loss‚2.9315623711731473 .loss/FBt1/detection_loss‚1.472340127453208 &loss/total_loss‚18.331045957212936 trainer/global_step‚8349 2loss/SRqqt2/assignment_loss‚0.8759281760563649 3loss/FBt2/assignment_loss‚0.0010538839370025717 /loss/FRt1/detection_loss‚2.1936558676814593 .loss/FRt2/detection_loss‚3.144033498178094 epoch‚0 % _timestamp‚1.7858137670194595e+09 _runtime‚3512.209223255 2loss/SRqqt1/assignment_loss‚0.8137798212813085 1loss/SRqqt1/detection_loss‚1.1471256876064542Ë•¨>À‰ 3loss/FBt1/assignment_loss‚3.585129127928025e-05 1loss/SRqqt2/detection_loss‚1.8843254211048286 /loss/FBt2/detection_loss‚1.6070876019696394 _step‚667 0loss/FRt1/assignment_loss‚2.2601176494801485 0loss/FRt2/assignment_loss‚2.9315623711731473 .loss/FBt1/detection_loss‚1.472340127453208 &loss/total_loss‚18.331045957212936 trainer/global_step‚8349 2loss/SRqqt2/assignment_loss‚0.8759281760563649 3loss/FBt2/assignment_loss‚0.0010538839370025717 /loss/FRt1/detection_loss‚2.1936558676814593 .loss/FRt2/detection_loss‚3.144033498178094 epoch‚0 % _timestamp‚1.7858137670194595e+09 _runtime‚3512.209223255 2loss/SRqqt1/assignment_loss‚0.8137798212813085 1loss/SRqqt1/detection_loss‚1.1471256876064542›Kö¥‘Á‚(:† “¶ÅÓ㜌ƒ gpu.0.gpu‚29.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚124.674' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚41.558 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0þÍ9ìi‚(:Þ  “¶ÅÓ«æÔƒ disk.drbd1550.in‚0* proc.memory.percent‚17.172330617904663 network.sent‚ 7.782153e+06 disk.drbd2041.out‚0 disk.rbd0.out‚0 disk.drbd1552.in‚0 proc.cpu.threads‚29 disk.drbd1356.in‚0 disk.drbd1550.out‚0$ memory_percent‚41.24188423156738 disk.sda.out‚ 137.2734375 disk.drbd1285.out‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.sda2.in‚11.625 disk.drbd2078.out‚0 disk.drbd2118.in‚0' proc.memory.availableMB‚ 38507.71875 network.recv‚529131* disk./.usagePercent‚32.805808018608246 disk.sda2.out‚ 137.2734375 disk.rbd1.in‚0 disk.drbd2078.in‚0 disk.sda1.in‚0$ disk./.usageGB‚586.6011657714844 disk.rbd1.out‚13.65283203125" disk.dm-0.out‚1007.72802734375 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0 disk.drbd1828.in‚0$ proc.memory.rssMB‚11254.05859375 disk.sda.in‚11.625 disk.dm-0.in‚ 423.734375 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.drbd2041.in‚0 cpu‚13.753381935679807% disk.nvme0n1.out‚1007.72802734375 disk.drbd1931.out‚0 disk.drbd1285.in‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 disk.drbd1226.in‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.drbd1965.in‚0 disk.nvme0n1.in‚ 423.734375 disk.drbd1584.out‚0A”ü2Ã- _wandbruntime‚3532  _runtime‚3532…Ý’ÇÄÁ & lr-AdamW/pg1‚0.0002164605942900859 & lr-AdamW/pg2‚0.0002164605942900859 trainer/global_step‚8399 % _timestamp‚1.7858137871404417e+09 _runtime‚3532.330480335 _step‚668/ဠÌÅÆ _step‚668 & lr-AdamW/pg1‚0.0002164605942900859 & lr-AdamW/pg2‚0.0002164605942900859 trainer/global_step‚8399 % _timestamp‚1.7858137871404417e+09 _runtime‚3532.330480335œÝ Ô#2Æ- _wandbruntime‚3532  _runtime‚3532)™óáÑ'ÇËO GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚140622009204832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚144545630358528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7590840026112 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55805777 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55805777 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚88710641 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚151596386480128 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 144516420 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚140621992165472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7372944548888 $ _timestamp‚1.785813787141483e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚151596386480128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚144205334907904 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33073195 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7590970700288 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24061070 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚144205334907904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55805779 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57134512 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚88710641 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚144545605454848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55805779 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7391034532864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57134265 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚88700985 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚151596282011136 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚147994953753720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24061315 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33073197 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7390947103232ašÖ'ÈÐO VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚144545605454848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55805779 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7391034532864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57134265 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚88700985 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚151596282011136 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚147994953753720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24061315 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33073197 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7390947103232 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚140622009204832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚144545630358528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7590840026112 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55805777 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55805777 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚88710641 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚151596386480128 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 144516420 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚140621992165472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚247 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7372944548888 $ _timestamp‚1.785813787141483e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚151596386480128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚144205334907904 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33073195 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7590970700288 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24061070 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚144205334907904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55805779 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57134512 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚88710641 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚862781448¡Y˜2É- _wandbruntime‚3532  _runtime‚3532 5éa•ÊÕM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚88721178 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚144562837475328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7373736218986 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57141114 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33077134 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚144222501670912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33077132 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55812422 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚3532.687518442 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚151614432869888 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 144523928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚148012468556778 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚88711508 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24064234 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚140638732337792 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚144562812571648 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55812420 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55812420 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚144222518710272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7391914159616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚152154713927680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7591876452352 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚88711508 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7391826722816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _step‚670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24063982 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7373822022222 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚152154554162688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_bþ!HF atch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55812422 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57141368 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚140638749377152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7391826722816 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚144222501670912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚8399 $ _timestamp‚1.785813787497209e+09 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 144523928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚144222518710272 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚88721178 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 144533600 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7591741591040 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚148012571399374 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚144222518710272 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚88721178 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 144533600 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7591741591040 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚148012571399374 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57141114 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33077134 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚144222501670912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33077132 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55812422 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 _runtime‚3532.687518442 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚151614432869888 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 144523928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚148012468556778 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚88711508 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24064234 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚140638732337792 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚144562812571648 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55812420 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55812420 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚144222518710272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7391914159616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚152154713927680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7591876452352 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚88711508 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7391826722816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _step‚670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24063982 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7373822022222 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚152154554162688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55812422 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57141368 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚254 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚140638749377152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7391826722816 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚144222501670912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚8399 $ _timestamp‚1.785813787497209e+09 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392ž¤8òG2Ì- _wandbruntime‚3532  _runtime‚3532t~¸ Í„ /loss/FRt2/detection_loss‚2.3527120100824455 _runtime‚3532.688312331 _step‚671 /loss/FRt1/assignment_loss‚2.566453597762368 3loss/FBt1/assignment_loss‚2.133591673926714e-05 /loss/FRt1/detection_loss‚2.7051391757347365 2loss/SRqqt1/assignment_loss‚1.0048034031489423 1loss/SRqqt2/assignment_loss‚1.177066996290877 4loss/FBt2/assignment_loss‚3.2287696900620437e-06 1loss/SRqqt2/detection_loss‚1.8594261037336814 /loss/FBt2/detection_loss‚0.5364914652758411 &loss/total_loss‚17.390631338124756 trainer/global_step‚8399 0loss/FRt2/assignment_loss‚2.8327570294079027 1loss/SRqqt1/detection_loss‚1.0881090489412217 .loss/FBt1/detection_loss‚1.267647943060313 epoch‚0 % _timestamp‚1.7858137874984322e+09lüQΉ 2loss/SRqqt1/assignment_loss‚1.0048034031489423 1loss/SRqqt2/assignment_loss‚1.177066996290877 4loss/FBt2/assignment_loss‚3.2287696900620437e-06 1loss/SRqqt2/detection_loss‚1.8594261037336814 /loss/FBt2/detection_loss‚0.5364914652758411 &loss/total_loss‚17.390631338124756 trainer/global_step‚8399 0loss/FRt2/assignment_loss‚2.8327570294079027 1loss/SRqqt1/detection_loss‚1.0881090489412217 .loss/FBt1/detection_loss‚1.267647943060313 epoch‚0 % _timestamp‚1.7858137874984322e+09 /loss/FRt2/detection_loss‚2.3527120100824455 _runtime‚3532.688312331 _step‚671 /loss/FRt1/assignment_loss‚2.566453597762368 3loss/FBt1/assignment_loss‚2.133591673926714e-05 /loss/FRt1/detection_loss‚2.7051391757347365Ÿ)Õp³œÏ‚(:‘ ¢¶ÅÓ –ø‚ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚129.008' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.00266666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚09KËbЂ(:×  ¢¶ÅÓ„÷¼ƒ disk.drbd1226.in‚0 network.recv‚531595 disk.drbd2118.in‚0 disk.drbd1931.out‚0 disk.drbd2111.out‚0 disk.drbd1289.out‚0 proc.cpu.threads‚29 disk.drbd1285.out‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.drbd1931.in‚0 disk.nvme0n1.out‚ 1013.140625 disk.rbd0.out‚0 disk.nvme0n1.in‚ 423.734375 disk.drbd1828.out‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.sda.in‚11.625 disk.drbd2078.in‚0 disk.dm-0.out‚ 1013.140625 disk.drbd1584.out‚0 disk.drbd1965.in‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd2041.out‚0 disk.dm-0.in‚ 423.734375% memory_percent‚41.242462396621704* proc.memory.percent‚17.172330617904663 disk.sda2.out‚ 140.0546875 disk.rbd1.out‚13.65283203125 disk.drbd1356.in‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0$ proc.memory.rssMB‚11254.05859375 disk.sda2.in‚11.625 disk.sda1.in‚0 network.sent‚ 7.827234e+06 disk.drbd1356.out‚0 disk.drbd1550.in‚0 disk.drbd1285.in‚0 cpu‚13.753161200559823) disk./.usagePercent‚32.80583532585188 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 disk.drbd1550.out‚0 disk.drbd1510.out‚0 disk.sda1.out‚0* proc.memory.availableMB‚38507.33984375 disk.sda.out‚ 140.0546875 disk.drbd1673.out‚0$ disk./.usageGB‚586.6016540527344 disk.drbd1817.out‚0 disk.drbd1289.in‚0˹2Ñ- _wandbruntime‚3553  _runtime‚3553iΪ¸ÉÒà _runtime‚3553.227328715 _step‚672 ' lr-AdamW/pg1‚0.00021774920361435123 ' lr-AdamW/pg2‚0.00021774920361435123 trainer/global_step‚8449 % _timestamp‚1.7858138080373855e+09VÑN>ÎÓÈ ' lr-AdamW/pg1‚0.00021774920361435123 ' lr-AdamW/pg2‚0.00021774920361435123 trainer/global_step‚8449 % _timestamp‚1.7858138080373855e+09 _runtime‚3553.227328715 _step‚672 W ­Ì2Ô- _wandbruntime‚3553  _runtime‚3553Šf£ÚÒ'ÕÌO PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚89238643 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24204496 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚153042145857024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚148875747990744 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56137927 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚145063690097664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 145376572 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚152498720765952 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56137929 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57474801 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7636026887168 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33270045 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚152498616296960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7434943238656 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚89238643 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7434943238656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24204754 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚145063673058304 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7416747209272 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 trainer/global_step‚8449 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7636159658496 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 145366914 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚141459000781472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7416833012396 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145063673058304 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚89228987 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33270047 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56137927 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚152498616296960 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚145405986198528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7435030668288 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚89228987 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚148875850833228 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032òb;ÈiÖÑO ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7435030668288 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚89228987 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚148875850833228 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚89238643 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24204496 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚153042145857024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚148875747990744 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56137927 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚145063690097664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 145376572 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚152498720765952 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56137929 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57474801 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7636026887168 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33270045 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚152498616296960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7434943238656 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚89238643 E?DeviceStatsãß0™nMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7434943238656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24204754 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚145063673058304 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7416747209272 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 trainer/global_step‚8449 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7636159658496 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 145366914 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚141459000781472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7416833012396 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145063673058304 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚89228987 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33270047 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56137927 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚152498616296960 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚145405986198528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0¡#W4†2×- _wandbruntime‚3553  _runtime‚3553å3x¢Ü&ØÖM O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚145080839821312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚89249180 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚89239510 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚145423168411648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7417710485730 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚89249180 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7435910295040 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33273984 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚227 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚153060252622848 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚153060103343616 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145080839821312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 145384080 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56144572 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7637059307520 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚141475740953792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56144570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 145384080 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145080856860672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚89239510 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7636934931968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚148893365636286 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚3553.582985838 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7417624682494 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56144572 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚229 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24207305 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24207532 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7435822858240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 145393752 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7435822858240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 145393752 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57481287 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚152516767155712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚152516662679552 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚141475757993152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7417624682494 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56144572 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚229 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24207305 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24207532 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7435822858240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 145393752 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7435822858240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 145393752 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57481287 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚152516767155712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚152516662679552 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚141475757993152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7435910295040 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33273984 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚227 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚153060252622848 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚153060103343616 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145080839821312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 145384080 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56144572 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7637059307520 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚141475740953792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56144570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 145384080 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145080856860672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚89239510 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7636934931968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚148893365636286 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚3553.582985838 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121¢¡£xø2Ú- _wandbruntime‚3553  _runtime‚3553ªÉå£Ûÿ epoch‚0 _runtime‚3553.583711978 _step‚675 0loss/FRt1/assignment_loss‚2.1861659466330683 0loss/FRt2/assignment_loss‚3.0940401438743836 3loss/FBt2/assignment_loss‚0.0002751079832766981 1loss/SRqqt1/detection_loss‚0.9864842306618413 1loss/SRqqt2/detection_loss‚2.0944096984511074 /loss/FBt2/detection_loss‚1.1728902786552649 trainer/global_step‚8449 .loss/FRt1/detection_loss‚2.596776940484545 -loss/FRt2/detection_loss‚2.47400204917436 &loss/total_loss‚18.890514324494056 2loss/SRqqt1/assignment_loss‚0.9788169172944896 % _timestamp‚1.7858138083939178e+09 2loss/SRqqt2/assignment_loss‚1.0801076109267094 1loss/FBt1/assignment_loss‚0.10010947172622675 -loss/FBt1/detection_loss‚2.12643592862878ÌI•Q Ü„ _runtime‚3553.583711978 _step‚675 0loss/FRt1/assignment_loss‚2.1861659466330683 0loss/FRt2/assignment_loss‚3.0940401438743836 3loss/FBt2/assignment_loss‚0.0002751079832766981 1loss/SRqqt1/detection_loss‚0.9864842306618413 1loss/SRqqt2/detection_loss‚2.0944096984511074 /loss/FBt2/detection_loss‚1.1728902786552649 trainer/global_step‚8449 .loss/FRt1/detection_loss‚2.596776940484545 -loss/FRt2/detection_loss‚2.47400204917436 &loss/total_loss‚18.890514324494056 2loss/SRqqt1/assignment_loss‚0.9788169172944896 % _timestamp‚1.7858138083939178e+09 2loss/SRqqt2/assignment_loss‚1.0801076109267094 1loss/FBt1/assignment_loss‚0.10010947172622675 -loss/FBt1/detection_loss‚2.12643592862878 epoch‚0£MØ4›Ý‚(: ±¶ÅÓșƃ gpu.0.gpu‚32.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.052' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.01733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ç‰]XhÞ‚(:Ý  ±¶ÅÓ‚ºÔƒ disk.drbd1285.in‚0 disk.drbd1289.out‚0 disk.drbd1673.in‚0$ disk.nvme0n1.out‚1016.2080078125 disk.dm-0.in‚ 423.734375 disk.drbd1673.out‚0 disk.drbd1965.in‚0) disk./.usagePercent‚32.80584300601414 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.rbd1.in‚0 network.recv‚534123! disk.dm-0.out‚1016.2080078125 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.nvme0n1.in‚ 423.734375 disk.drbd1356.out‚0 disk.drbd1226.out‚0$ disk./.usageGB‚586.6017913818359 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.sda1.out‚0 disk.drbd1931.out‚0 disk.sda1.in‚0 disk.drbd2118.in‚0 proc.cpu.threads‚29 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0* proc.memory.percent‚17.172330617904663 disk.sda2.in‚11.625 cpu‚13.752628397515965 disk.drbd1285.out‚0 disk.sda.in‚11.625$ memory_percent‚41.24273657798767 disk.drbd2041.in‚0 network.sent‚ 7.872472e+06 disk.drbd1817.out‚0 disk.sda2.out‚ 140.3359375 disk.drbd1584.in‚0 disk.drbd1938.out‚0 disk.drbd1356.in‚0 disk.sda.out‚ 140.3359375 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.drbd2041.out‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.rbd1.out‚ 13.7158203125 disk.drbd1931.in‚0* proc.memory.availableMB‚38507.16015625 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.drbd1817.in‚0 disk.drbd1584.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1550.out‚0)m«œß‚(:‘ À¶ÅÓ˜„£ƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚107.275' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚35.75833333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0«Dýmà‚(:â  À¶ÅÓÍçƒ$ proc.memory.rssMB‚11254.05859375 disk.drbd1289.in‚0 disk.rbd1.in‚0* proc.memory.availableMB‚38506.91015625 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd1226.out‚0$ memory_percent‚41.24311804771423 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd1938.out‚0 disk.drbd1584.in‚0 disk.drbd2041.out‚0 cpu‚13.752341156037811 disk.drbd1817.in‚0 disk.rbd1.out‚ 13.7158203125 disk.drbd2041.in‚0 disk.drbd2111.out‚0 disk.sda2.out‚ 142.94140625 disk.drbd1226.in‚0 disk.drbd1965.out‚0 network.sent‚ 7.917559e+06 disk.drbd1828.in‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.rbd0.out‚0$ disk./.usageGB‚586.6056823730469 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0* disk./.usagePercent‚32.806060610611816 disk.drbd1550.in‚0 disk.nvme0n1.in‚ 423.734375 disk.drbd2078.in‚0 disk.drbd2078.out‚0 network.recv‚536561 disk.drbd1285.out‚0 disk.drbd1510.in‚0% disk.nvme0n1.out‚1025.79248046875 disk.drbd1931.in‚0 disk.sda.in‚11.625 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.dm-0.in‚ 423.734375" disk.dm-0.out‚1025.79248046875 disk.sda1.out‚0 disk.drbd1356.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd1285.in‚0 disk.sda2.in‚11.625 disk.drbd1356.in‚0 disk.sda.out‚ 142.94140625 disk.drbd1510.out‚0 disk.drbd2118.out‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0 proc.cpu.threads‚29 disk.drbd1938.in‚0ò¸E·2á- _wandbruntime‚3573  _runtime‚3573ÛÏ­ÇâÁ & lr-AdamW/pg1‚0.0002190378129386165 & lr-AdamW/pg2‚0.0002190378129386165 trainer/global_step‚8499 % _timestamp‚1.7858138287495732e+09 _runtime‚3573.939533547 _step‚676YŒeÌãÆ & lr-AdamW/pg1‚0.0002190378129386165 & lr-AdamW/pg2‚0.0002190378129386165 trainer/global_step‚8499 % _timestamp‚1.7858138287495732e+09 _runtime‚3573.939533547 _step‚676¤n!îh2ä- _wandbruntime‚3573  _runtime‚3573DDkŽïåÌO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145922011208704 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚153400950582784 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7681374105600 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33466895 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚145922028248064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7478939374080 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57815288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_poߎã!ol.freed‚146266317134848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56470079 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚89766645 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56470077 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚241 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56470079 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚89756989 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7478939374080 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 146236724 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚146266342038528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7681239237120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚145922011208704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24348393 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 trainer/global_step‚8499 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24348632 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚89756989 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 146227066 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚153947716144128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚142296026436832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚89766645 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚3573.943622686 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚153401055051776 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _step‚677 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚149756747912736 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚153401055051776 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7479026803712 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7460635672780 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚153400950582784 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7479026803712 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚239 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33466897 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚145922028248064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56470077 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚153947556371968 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57815529 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858138287505057e+09’•Ï×'æÑO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858138287505057e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145922011208704 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚153400950582784 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7681374105600 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33466895 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚145922028248064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7478939374080 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57815288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚146266317134848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56470079 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚89766645 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56470077 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚241 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56470079 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚89756989 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7478939374080 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 146236724 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚146266342038528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7681239237120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚145922011208704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24348393 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 trainer/global_step‚8499 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24348632 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚89756989 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 146227066 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚153947716144128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚142296026436832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚89766645 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚3573.943622686 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚153401055051776 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _step‚677 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚149756747912736 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚153401055051776 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7479026803712 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7460635672780 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚153400950582784 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7479026803712 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚239 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33466897 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚145922028248064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56470077 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚153947556371968 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57815529¥*eW]2ç- _wandbruntime‚3574  _runtime‚3574Wµ«Û&èÕM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 _runtime‚3574.293529082 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57822056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33470832 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7479906430464 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚142312749569792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚263 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 146244232 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145939177971712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24351487 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚153965825031680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚8499 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7479818993664 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚153419101441536 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7682275876352 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7682134723584 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚153965658975232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145939195011072 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚146283524251648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7461513146002 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚149774365558390 $ _timestamp‚1.785813829103232e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚89767512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7479818993664 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 146253904 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56476720 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚89777182š9r—éÚM _runtime‚3574.293529082 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57822056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33470832 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7479906430464 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚142312749569792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚263 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 146244232 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145939177971712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24351487 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \HDeviceStatsMonitor.o¿‡Ñ$Ên_train_batch_end/inactive_split_bytes.all.allocated‚153965825031680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚8499 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7479818993664 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚153419101441536 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7682275876352 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7682134723584 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚153965658975232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145939195011072 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚146283524251648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7461513146002 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚149774365558390 $ _timestamp‚1.785813829103232e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚89767512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7479818993664 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 146253904 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56476720 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚89777182 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856¦¨‘#2ê- _wandbruntime‚3574  _runtime‚3574‚?¨wë€ 4loss/FBt2/assignment_loss‚6.0328507948221545e-05 &loss/total_loss‚17.922459785140557 $ _timestamp‚1.785813829104416e+09 _runtime‚ 3574.29420615 _step‚679 epoch‚0 /loss/FRt1/assignment_loss‚2.818511342058087 1loss/SRqqt2/assignment_loss‚1.176145223189484 1loss/SRqqt2/detection_loss‚1.8598677260293202 .loss/FRt2/detection_loss‚2.303267475470756 /loss/FBt2/detection_loss‚0.6510822218443666 trainer/global_step‚8499 /loss/FBt1/detection_loss‚2.1789982161269736 0loss/FRt2/assignment_loss‚2.4237955675541776 1loss/SRqqt1/assignment_loss‚1.150188721783891 3loss/FBt1/assignment_loss‚0.0015389101017979977 .loss/FRt1/detection_loss‚2.312613617665697 1loss/SRqqt1/detection_loss‚1.0463904348080573šlWq ì… .loss/FRt2/detection_loss‚2.303267475470756 /loss/FBt2/detection_loss‚0.6510822218443666 trainer/global_step‚8499 /loss/FBt1/detection_loss‚2.1789982161269736 0loss/FRt2/assignment_loss‚2.4237955675541776 1loss/SRqqt1/assignment_loss‚1.150188721783891 3loss/FBt1/assignment_loss‚0.0015389101017979977 .loss/FRt1/detection_loss‚2.312613617665697 1loss/SRqqt1/detection_loss‚1.0463904348080573 4loss/FBt2/assignment_loss‚6.0328507948221545e-05 &loss/total_loss‚17.922459785140557 $ _timestamp‚1.785813829104416e+09 _runtime‚ 3574.29420615 _step‚679 epoch‚0 /loss/FRt1/assignment_loss‚2.818511342058087 1loss/SRqqt2/assignment_loss‚1.176145223189484 1loss/SRqqt2/detection_loss‚1.8598677260293202§̵Öêí‚(:… ϶ÅÓ¤¨˜ƒ gpu.0.gpu‚34.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.732' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.244 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0µä’Þdî‚(:Ù  ϶ÅÓ™Ôàƒ network.recv‚537616 disk.drbd1510.out‚0 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.drbd1356.in‚0 disk.drbd1285.in‚0* proc.memory.availableMB‚38506.84765625* proc.memory.percent‚17.172330617904663 proc.cpu.threads‚29 disk.drbd2118.in‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.sda.out‚ 145.48046875 disk.drbd1552.out‚0 cpu‚13.752092067261513 disk.drbd1289.in‚0 disk.drbd1584.out‚0 disk.sda1.in‚0 disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.drbd1550.out‚0 disk.drbd2111.out‚0 disk.drbd1226.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1289.out‚0 disk.nvme0n1.in‚ 423.734375 disk.drbd1226.out‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.dm-0.out‚ 1034.6484375 disk.drbd1673.in‚0 disk.sda2.out‚ 145.48046875 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd1584.in‚0 disk.rbd0.in‚0 network.sent‚ 7.920595e+06 disk.drbd1552.in‚0 disk.sda2.in‚11.625) disk./.usagePercent‚32.80604461027375 disk.dm-0.in‚ 423.734375 disk.sda1.out‚0! disk.nvme0n1.out‚ 1034.6484375 disk.drbd1285.out‚0 disk.drbd1673.out‚0 disk.sda.in‚11.625# disk./.usageGB‚586.605396270752 disk.rbd1.out‚ 13.7158203125 disk.drbd1965.in‚0 disk.drbd2078.in‚0% memory_percent‚41.243213415145874 )ü±2ï- _wandbruntime‚3594  _runtime‚3594n2ŒÉðà _runtime‚3594.717079764 _step‚680 ' lr-AdamW/pg1‚0.00022032642226288185 ' lr-AdamW/pg2‚0.00022032642226288185 trainer/global_step‚8549 % _timestamp‚1.7858138495271428e+09 ~ÎñÈ trainer/global_step‚8549 % _timestamp‚1.7858138495271428e+09 _runtime‚3594.717079764 _step‚680 ' lr-AdamW/pg1‚0.00022032642226288185 ' lr-AdamW/pg2‚0.00022032642226288185¨õW2ò- _wandbruntime‚3594  _runtime‚3594@8ÌÑ'óËO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56802229 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚147126697878528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56802227 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24491724 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7726530932736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚154303284868608 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33663745 _step‚681 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7522935509504 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚154853228811264 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚150637542149760 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7522935509504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚90294647 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚147126672974848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚154303389337600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56802227 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚146780349359104 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7504609939412 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58155707 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚150637644992244 trainer/global_step‚8549 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7523022939136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚154853073233408 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚90294647 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33663747 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 147096876 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚154303284868608 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56802229 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚236 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 147087218 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚154303389337600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚238 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7726400258560 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 $ _timestamp‚1.785813849528065e+09 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58155469 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚143133035052832 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚146780349359104 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚90284991 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚146780366398464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚143133018013472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7523022939136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚146780366398464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚146780366398464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚143133018013472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7523022939136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚146780366398464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7523022939136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚154853073233408 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚90294647 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33663747 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 147096876 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚154303284868608 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56802229©ÞË_2õ- _wandbruntime‚3595  _runtime‚3595÷yMÉöÖM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7523815129088 % _timestamp‚1.7858138498831308e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 147104384 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 147114056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚143149775225152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 147114056 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚3595.073455722 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ICDeviceStatsMonitor.on_train_b‡¹+catch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚154871340369408 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚143149758185792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56808872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚90305184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33667682 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚150655159795302 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56808870 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58162224 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33667684 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24494798 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚147143880091648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7523815129088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚90295514 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56808872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚154871182701568 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚143149775225152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 147114056 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 _runtime‚3595.073455722 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚154871340369408 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚143149758185792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56808872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚90305184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33667682 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚150655159795302 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56808870 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58162224 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33667684 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24494798 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚147143880091648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7523815129088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚90295514 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56808872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚154871182701568 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚90822649 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚685 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚155205619154432 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚155205723623424 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚147638704548864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57134379 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58495919 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚155205723623424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7771568208384 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚155205619154432 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7567019074560 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚90812993 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚147638704548864 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚90822649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7858138707123203e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7771696785408 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚143970043668832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 147947370 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚143970026629472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7566931644928 PC`Çd eDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57134377 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58495665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 147957028 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7566931644928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24635070 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚90822649 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚685 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚155205619154432 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚155205723623424 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚147638704548864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57134379 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58495919 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚155205723623424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7771568208384 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚155205619154432 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7567019074560 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚90812993 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚147638704548864 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚90822649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7858138707123203e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7771696785408 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚143970043668832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 147947370 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚143970026629472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7566931644928 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57134377 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58495665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 147957028 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7566931644928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24635070 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚90823516 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚147655854272512 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚147655871311872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7567898701312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7567811264512 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7549290073018 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚90833186 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚8599 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 147964536 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33864534 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 147964536 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148004235931648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7567811264512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58502458 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57141022 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚155223770013184 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148004260835328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚151536159717406 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚143986766801792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7549375876254 _runtime‚3616.254040745 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚155223665537024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚147655871311872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7567898701312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57141020 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚155776865216000 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7772467422208 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚90833186 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57141022 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 147974208 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251x–âà&‡ÚM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57141020 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚90823516 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚147655854272512 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚147655871311872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7567898701312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7567811264512 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7549290073018 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚90833186 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 trainer/global_step‚8599 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 147964536 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33864534 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 147964536 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148004235931648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7567811264512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58502458 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57141022 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚155223770013184 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148004260835328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚151536159717406 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚143986766801792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7549375876254 _runtime‚3616.254040745 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚155223665537024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚147655871311872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7567898701312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57141020 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚155776865216000 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7772467422208 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚90833186 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57141022 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 147974208 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596®ÞŸóú2ˆ- _wandbruntime‚3616  _runtime‚3616þ.ú ‰ƒ /loss/FRt1/assignment_loss‚2.283647501107418 2loss/SRqqt1/assignment_loss‚0.9253145262724753 1loss/SRqqt2/detection_loss‚2.3032281669974326 /loss/FBt1/detection_loss‚1.6814790007192641 2loss/FBt1/assignment_loss‚0.008107087021713255 /loss/FRt2/detection_loss‚2.5547017318656646 epoch‚0 % _timestamp‚1.7858138710649753e+09 _runtime‚3616.254780047 2loss/SRqqt2/assignment_loss‚0.9288769635558128 1loss/SRqqt1/detection_loss‚1.0326339241905804 trainer/global_step‚8599 _step‚687 0loss/FRt2/assignment_loss‚3.0158681065203194 2loss/FBt2/assignment_loss‚0.043894986311582805 /loss/FRt1/detection_loss‚2.6066807774883327 /loss/FBt2/detection_loss‚1.0440840752368483 &loss/total_loss‚18.428516847287444R¾K%dŠˆ trainer/global_step‚8599 _step‚687 0loss/FRt2/assignment_loss‚3.015868106520319îããª4 2loss/FBt2/assignment_loss‚0.043894986311582805 /loss/FRt1/detection_loss‚2.6066807774883327 /loss/FBt2/detection_loss‚1.0440840752368483 &loss/total_loss‚18.428516847287444 /loss/FRt1/assignment_loss‚2.283647501107418 2loss/SRqqt1/assignment_loss‚0.9253145262724753 1loss/SRqqt2/detection_loss‚2.3032281669974326 /loss/FBt1/detection_loss‚1.6814790007192641 2loss/FBt1/assignment_loss‚0.008107087021713255 /loss/FRt2/detection_loss‚2.5547017318656646 epoch‚0 % _timestamp‚1.7858138710649753e+09 _runtime‚3616.254780047 2loss/SRqqt2/assignment_loss‚0.9288769635558128 1loss/SRqqt1/detection_loss‚1.0326339241905804¯°‹¨¤‹‚(:’ ü¶ÅÓګ˃ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚124.018' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚41.339333333333336 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ô#ì‡kŒ‚(:à  ü¶ÅÓüÅàƒ disk.drbd1550.out‚0 disk.drbd1828.out‚0 disk.sda1.in‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚ 423.734375 disk.drbd1584.out‚0 disk.drbd1285.out‚0 disk.sda1.out‚0 disk.drbd1931.in‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 network.recv‚544160% disk.nvme0n1.out‚1811.80615234375 disk.drbd1356.out‚0 disk.rbd0.out‚0% memory_percent‚41.242969036102295* proc.memory.percent‚17.172330617904663 disk.drbd1931.out‚0 proc.cpu.threads‚29) disk./.usagePercent‚32.94378960062337 disk.sda.out‚ 153.46484375) proc.memory.availableMB‚ 38507.0078125$ proc.memory.rssMB‚11254.05859375 disk.drbd1673.in‚0 disk.drbd2041.out‚0 disk.drbd1289.in‚0 disk.drbd1226.in‚0 disk.drbd1510.in‚0 disk.drbd1584.in‚0" disk.dm-0.out‚1811.80615234375 cpu‚13.750915769601848 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.sda2.in‚11.625 disk.drbd1356.in‚0 disk.drbd1965.out‚0 network.sent‚ 8.014004e+06 disk.drbd1938.out‚0 disk.drbd1510.out‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1828.in‚0# disk./.usageGB‚589.068416595459 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.drbd2118.in‚0 disk.rbd1.in‚0 disk.drbd1965.in‚0 disk.drbd2111.out‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 153.46484375 disk.sda.in‚11.625 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.dm-0.in‚ 423.734375 disk.rbd1.out‚ 13.8642578125 disk.drbd2041.in‚0½eªó2- _wandbruntime‚3636  _runtime‚3636×ÑÇŽÁ ' lr-AdamW/pg1‚0.00022290364091141242 ' lr-AdamW/pg2‚0.00022290364091141242 trainer/global_step‚8649 # _timestamp‚1.78581389168064e+09 _runtime‚3636.870599857 _step‚688p¥:ÌÆ ' lr-AdamW/pg1‚0.00022290364091141242 ' lr-AdamW/pg2‚0.00022290364091141242 trainer/global_step‚8649 # _timestamp‚1.78581389168064e+09 _runtime‚3636.870599857 _step‚688°BMY2- _wandbruntime‚3636  _runtime‚3636i‰­Ñ'‘ËO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57466527 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚156664108481024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚144807052284832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚148497025659904 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7611015209984 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58836012 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚152399439151260 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚144807035245472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57466529 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚156108057909248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7611015209984 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚156107953440256 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚156107953440256 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚148847384654848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7816723826176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 148807522 $ _timestamp‚1.785813891681564e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7610927780352 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58836269 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7592386866428 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7610927780352 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24778567 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚148847409558528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚91340995 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚148497025659904 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚91350651 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392³¨?†Ö'’ÐO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7610927780352 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58836269 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7592386866428 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7610927780352 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24778567 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚148847409558528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚91340995 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚148497025659904 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚91350651 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57466527 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚156664108481024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚144807052284832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚148497025659904 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7611015209984 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58836012 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚152399439151260 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚144807035245472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57466529 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚156108057909248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7611015209984 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚156107953440256 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚156107953440256 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚148847384654848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚257 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7816723826176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 148807522 $ _timestamp‚1.785813891681564e+09±æ1 M2“- _wandbruntime‚3637  _runtime‚3637¡‹1#”ÖM _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148864591771648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚144823792457152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚91361188 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7611894836736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58842688 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148864616675328 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 _step‚690 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚8649 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚148514192422912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24781558 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7593264339762 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156125999822848 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7593178536526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57473172 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34061384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156126104299008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚144823775417792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57473172 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58842942 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚152417056796914 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 148824688 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚148514209462272 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7611807399936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/activejrq0«_bytes.all.allocated‚156126104299008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚148514209462272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 148834360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57473170 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236ÆÎá&•ÛM TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57473172 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58842942 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚152417056796914 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 148824688 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚148514209462272 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7611807399936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚156126104299008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚148514209462272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 148834360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57473170 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148864591771648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚144823792457152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚91361188 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7611894836736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58842688 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148864616675328 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 _step‚690 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚8649 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚148514192422912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24781558 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7593264339762 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156125999822848 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7593178536526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57473172 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34061384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156126104299008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚144823775417792²z¨ ’2–- _wandbruntime‚3637  _runtime‚3637t3-—‚ 1loss/SRqqt1/detection_loss‚0.9856902844017479 1loss/SRqqt2/detection_loss‚1.8607483958252347 /loss/FBt1/detection_loss‚2.5487821423282315 /loss/FBt2/detection_loss‚0.6838766746713143 _step‚691 2loss/FBt1/assignment_loss‚0.003107955708567748 &loss/total_loss‚18.777399384337436 2loss/SRqqt1/assignment_loss‚1.0397756345033242 _runtime‚3637.229654319 /loss/FRt1/assignment_loss‚2.538561668151464 0loss/FRt2/assignment_loss‚2.9181497369123544 2loss/FBt2/assignment_loss‚4.18540741395795e-05 .loss/FRt1/detection_loss‚2.479460494258465 /loss/FRt2/detection_loss‚2.4965441868357034 epoch‚0 trainer/global_step‚8649 % _timestamp‚1.7858138920398679e+09 2loss/SRqqt2/assignment_loss‚1.2226603566668928È®2| ˜‡ epoch‚0 trainer/global_step‚8649 % _timestamp‚1.7858138920398679e+09 2loss/SRqqt2/assignment_loss‚1.2226603566668928 1loss/SRqqt1/detection_loss‚0.9856902844017479 1loss/SRqqt2/detection_loss‚1.8607483958252347 /loss/FBt1/detection_loss‚2.5487821423282315 /loss/FBt2/detection_loss‚0.6838766746713143 _step‚691 2loss/FBt1/assignment_loss‚0.003107955708567748 &loss/total_loss‚18.777399384337436 2loss/SRqqt1/assignment_loss‚1.0397756345033242 _runtime‚3637.229654319 /loss/FRt1/assignment_loss‚2.538561668151464 0loss/FRt2/assignment_loss‚2.9181497369123544 2loss/FBt2/assignment_loss‚4.18540741395795e-05 .loss/FRt1/detection_loss‚2.479460494258465 /loss/FRt2/detection_loss‚2.4965441868357034³ƒŒh:œ™‚(:‘ ‹·ÅÓš¶ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚104.074' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚34.69133333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0t*J6fš‚(:Û  ‹·ÅÓĶփ disk.drbd1552.in‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 disk.drbd1226.out‚0 proc.cpu.threads‚29 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 154.1953125 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0$ proc.memory.rssMB‚11254.05859375 network.sent‚ 8.062457e+06 disk.sda2.in‚11.625 disk.rbd1.out‚ 13.896484375$ disk.nvme0n1.out‚1936.4794921875 disk.drbd2041.in‚0 disk.dm-0.in‚ 431.34375! disk.dm-0.out‚1936.4794921875 disk.drbd1817.out‚0 disk.drbd1584.in‚0 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.drbd2111.in‚0 network.recv‚552350 disk.drbd1289.in‚0 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.sda.in‚11.625) disk./.usagePercent‚32.98302861635289 disk.drbd1289.out‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1356.in‚0 disk.drbd1828.out‚0 disk.drbd1550.out‚0% memory_percent‚41.243404150009155 disk.drbd1550.in‚0 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.nvme0n1.in‚ 431.34375 disk.sda.out‚ 154.1953125* proc.memory.availableMB‚38506.72265625 cpu‚13.750541858575364 disk.drbd1965.in‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0* proc.memory.percent‚17.172330617904663$ disk./.usageGB‚589.7700500488281¸þ L2›- _wandbruntime‚3657  _runtime‚3657 Œà`Ȝ ' lr-AdamW/pg1‚0.00022419225023567776 ' lr-AdamW/pg2‚0.00022419225023567776 trainer/global_step‚8699 $ _timestamp‚1.785813912590353e+09 _runtime‚3657.780293787 _step‚692[%ShÍÇ ' lr-AdamW/pg1‚0.00022419225023567776 ' lr-AdamW/pg2‚0.00022419225023567776 trainer/global_step‚8699 $ _timestamp‚1.785813912590353e+09 _runtime‚3657.780293787 _step‚692´$g‹“2ž- _wandbruntime‚3657  _runtime‚36577Š(“Ò'ŸÌO QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59176859 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7861864504320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7858139125912719e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 149677332 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚157010392195072 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚91868997 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚157569758479872 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚91868997 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24922562 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚145644043861472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7636189526812 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _step‚693 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚153280336230768 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚91878653 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24922325 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 trainer/global_step‚8699 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚149355380849664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7636275329936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7861993081344 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚91878653 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚149707765398528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚149707740494848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚157010287726080 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57798677 _runtime‚3657.784629414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 149667674 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7655011345408 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57798679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚157010287726080 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚157010392195072 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34254297 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚149355380849664 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57798677 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7654923915776 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚149355363810304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7654923915776 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚157569604999168 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚145644060900832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0Ûr(Ñ ÑO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57798677 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚237 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7654923915776 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚149355363810304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7654923915776 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚157569604999168 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚145644060900832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59176859 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7861864504320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7858139125912719e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 149677332 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚157010392195072 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚91868997 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚157569758479872 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚91868997 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚239 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24922562 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚145644043861472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7636189526812 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _step‚693 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚153280336230768 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚91878653 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24922325 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 trainer/global_step‚8699 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚149355380849664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7636275329936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7861993081344 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚91878653 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚149707765398528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚149707740494848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚157010287726080 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57798677 _runtime‚3657.784629414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 149667674 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7655011345408 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57798679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚157010287726080 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚157010392195072 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34254297 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚149355380849664 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251µ·Ä—¶2¡- _wandbruntime‚3658  _runtime‚36586öPÜ&¢ÖM D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 149684840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚149372547612672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7655803535360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚145660801073152 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚91889190 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7655890972160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 % _timestamp‚1.7858139129553547e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚91879520 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚149724947611648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚153297851033826 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 149684840 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24925448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚157028438584832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚249 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 149694512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7862906116096 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57805322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7637152803270 _step‚694 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚149372530573312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59183431 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚149724972515328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57805320 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7655803535360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚91889190 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7637067000034 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚157587718866432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34258232 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚157028334108672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57805322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚149372547612672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚149372530573312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚145660784033792½ôá&£ÛM >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7655803535360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚91889190 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7637067000034 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚157587718866432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34258232 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚157028334108672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57805322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚149372547612672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚149372530573312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚145660784033792 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 149684840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚149372547612672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7655803535360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚145660801073152 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚91889190 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7655890972160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 % _timestamp‚1.7858139129553547e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚91879520 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚149724947611648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚153297851033826 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 149684840 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24925448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚157028438584832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚249 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 149694512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7862906116096 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57805322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7637152803270 _step‚694 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚149372530573312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59183431 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚149724972515328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57805320 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2¶+]c+e©‚(:Ú  ©·ÅÓÁŒ¬ƒ disk.drbd2078.in‚0 disk.drbd2118.out‚0 disk.drbd1356.in‚0 disk.sda.in‚11.625 network.sent‚ 8.152532e+06 disk.drbd1510.in‚0 disk.sda2.in‚11.625 disk.drbd1584.out‚0 network.recv‚555795 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.drbd1510.out‚0 disk.sda1.out‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0* proc.memory.percent‚17.172330617904663$ disk./.usageGB‚589.3503189086914 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.drbd1552.out‚0 disk.drbd1965.in‚0 disk.drbd1828.in‚0 disk.drbd1550.in‚0 disk.drbd2041.out‚0& proc.memory.availableMB‚ 38506.6875 disk.drbd1226.out‚0 disk.drbd1285.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.rbd0.out‚0 disk.sda2.out‚ 155.51171875 disk.rbd1.in‚0 disk.dm-0.in‚ 431.34375 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd1285.in‚0! disk.dm-0.out‚3913.7373046875$ memory_percent‚41.24345779418945 disk.sda1.in‚0) disk./.usagePercent‚32.95955505372427 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚ 431.34375 proc.cpu.threads‚29 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd1931.out‚0$ proc.memory.rssMB‚11254.05859375 cpu‚13.749938148683837 disk.rbd0.in‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd1931.in‚0 disk.sda.out‚ 155.51171875 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.rbd1.out‚13.95947265625$ disk.nvme0n1.out‚3913.7373046875©í-œª‚(:‘ ©·ÅÓ®¡¤ƒ gpu.0.gpu‚39.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚128.164' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.72133333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01’C2«- _wandbruntime‚3678  _runtime‚3678Þ+Uɬà ' lr-AdamW/pg1‚0.00022548085955994304 ' lr-AdamW/pg2‚0.00022548085955994304 trainer/global_step‚8749 % _timestamp‚1.7858139334375606e+09 _runtime‚3678.627502818 _step‚696RV›éÎ­È ' lr-AdamW/pg1‚0.00022548085955994304 ' lr-AdamW/pg2‚0.00022548085955994304 trainer/global_step‚8749 % _timestamp‚1.7858139334375606e+09 _runtime‚3678.627502818 _step‚696¸­ èÛ2®- _wandbruntime‚3678  _runtime‚3678zm”¯ËO LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚150213719000064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7699007480832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7680163793444 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚92396999 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚150568121238528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7698920051200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34451147 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _runtime‚ 3678.63162913 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 150527826 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58130829 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚150213701960704 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 N@DeviceStatsMonitor.on_train_batch_sîwƒ"Atart/allocation.all.allocated‚ 150537484 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚157912622011904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚157912622011904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7698920051200 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚92406655 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚246 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858139334384675e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚158475284533760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58130827 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚92406655 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7680077990320 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚158475124761600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚157912726480896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58130827 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59517171 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7699007480832 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25066272 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34451145 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚146481052477472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25066026 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58130829 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚697 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚150213701960704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0š› #Ö'°ÐO 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25066026 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58130829 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚697 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚150213701960704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚150213719000064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7699007480832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7680163793444 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚92396999 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚150568121238528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7698920051200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34451147 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 _runtime‚ 3678.63162913 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 150527826 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58130829 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚150213701960704 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 150537484 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚157912622011904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚157912622011904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7698920051200 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚92406655 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚246 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858139334384675e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚158475284533760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58130827 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚92406655 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7680077990320 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚158475124761600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚157912726480896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58130827 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59517171 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7699007480832 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25066272 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34451145 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚146481052477472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0¹5{Úø2±- _wandbruntime‚3678  _runtime‚3678T˜+ªÜ&²ÖM QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚157930772870656 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚92417192 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7681041266778 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚92407522 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 150544992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7907932465664 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚146497809689152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7699887107584 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚92407522 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58137472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7699799670784 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58137470 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚92417192 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25068910 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚158493235917312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚150230868723712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7699887107584 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 150554664 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚154178850955930 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚158493393585152 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59523992 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚698 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34455084 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚157930668394496 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚157930772870656 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚150230868723712 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚150230885763072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚154178748113334 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚146497792649792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 150544992 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7699799670784 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚260 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 % _timestamp‚1.7858139337928085e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7908065229824 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7680955463542 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 trainer/global_step‚8749 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59523992 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚698 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34455084 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚157930668394496 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚157930772870656 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚150230868723712 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚150230885763072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚154178748113334 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚146497792649792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 150544992 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7699799670784 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚260 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 % _timestamp‚1.7858139337928085e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7908065229824 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7680955463542 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 trainer/global_step‚8749 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚157930772870656 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚Ô7|)92417192 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7681041266778 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚92407522 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 150544992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7907932465664 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚146497809689152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7699887107584 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚92407522 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58137472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7699799670784 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58137470 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚92417192 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25068910 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚158493235917312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚150230868723712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7699887107584 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 150554664 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚154178850955930º©âq'2´- _wandbruntime‚3678  _runtime‚3678n®à µ… _runtime‚3678.983801854 &loss/total_loss‚18.294740704995462 0loss/FRt1/assignment_loss‚2.5396114921569826 /loss/FRt2/assignment_loss‚3.102814123381016 2loss/SRqqt1/assignment_loss‚0.9530692771938932 .loss/FRt1/detection_loss‚2.488346120317777 /loss/FRt2/detection_loss‚2.5806345064626184 3loss/FBt2/assignment_loss‚1.626401706141441e-05 /loss/FBt1/detection_loss‚2.0022316525823305 trainer/global_step‚8749 _step‚699 1loss/SRqqt2/detection_loss‚1.7418915507063937 /loss/FBt2/detection_loss‚1.0295103724541452 epoch‚0 2loss/SRqqt2/assignment_loss‚1.0457060516380996 4loss/FBt1/assignment_loss‚4.0171580540767794e-05 1loss/SRqqt1/detection_loss‚0.8108691225046074 % _timestamp‚1.7858139337940247e+09Íøh¶Š /loss/FBt1/detection_loss‚2.0022316525823305 trainer/global_step‚8749 _step‚699 1loss/SRqqt2/detection_loss‚1.7418915507063937 /loss/FBt2/detection_loss‚1.0295103724541452 epoch‚0 2loss/SRqqt2/assignment_loss‚1.0457060516380996 4loss/FBt1/assignment_loss‚4.0171580540767794e-05 1loss/SRqqt1/detection_loss‚0.8108691225046074 % _timestamp‚1.7858139337940247e+09 _runtime‚3678.983801854 &loss/total_loss‚18.294740704995462 0loss/FRt1/assignment_loss‚2.5396114921569826 /loss/FRt2/assignment_loss‚3.102814123381016 2loss/SRqqt1/assignment_loss‚0.9530692771938932 .loss/FRt1/detection_loss‚2.488346120317777 /loss/FRt2/detection_loss‚2.5806345064626184 3loss/FBt2/assignment_loss‚1.626401706141441e-05»žÄ¾‹›·‚(: ¸·ÅÓÿŸ†ƒ gpu.0.gpu‚34.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.275' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.09166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Å9Esc¸‚(:Ø  ¸·ÅÓÙòȃ) disk./.usagePercent‚32.95955100030529$ disk./.usageGB‚589.3502464294434 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd2041.out‚0 disk.drbd1550.in‚0 network.sent‚ 8.155565e+06 disk.drbd1285.out‚0 disk.drbd1673.in‚0% disk.nvme0n1.out‚3919.30517578125% proc.memory.availableMB‚ 38506.375" disk.dm-0.out‚3919.30517578125 disk.drbd1356.out‚0 disk.sda.in‚11.625 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.sda2.out‚ 157.671875 disk.rbd1.out‚13.95947265625 disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0* proc.memory.percent‚17.172330617904663 disk.dm-0.in‚ 431.34375 disk.drbd1510.in‚0 disk.drbd1938.out‚0 proc.cpu.threads‚29 disk.drbd1552.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚11.625 disk.drbd2078.in‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.rbd1.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2111.in‚0 disk.sda1.in‚0 disk.drbd1550.out‚0 disk.rbd0.in‚0% memory_percent‚41.243934631347656 disk.drbd1584.in‚0 cpu‚13.749636907536322 disk.drbd2118.in‚0 disk.drbd1817.in‚0 network.recv‚556709 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.drbd1510.out‚0 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1673.out‚0 disk.sda.out‚ 157.671875 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚ 431.34375 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.sda1.out‚0 disk.drbd1931.in‚0 disk.drbd1965.in‚0Å<2¹- _wandbruntime‚3699  _runtime‚3699`(D{ɺà ' lr-AdamW/pg1‚0.00022676946888420838 ' lr-AdamW/pg2‚0.00022676946888420838 trainer/global_step‚8799 % _timestamp‚1.7858139544225981e+09 _runtime‚3699.614630009 _step‚700œ±¤%Î»È _runtime‚3699.614630009 _step‚700 ' lr-AdamW/pg1‚0.00022676946888420838 ' lr-AdamW/pg2‚0.00022676946888420838 trainer/global_step‚8799 % _timestamp‚1.7858139544225981e+09¼Y¥ºÙ2¼- _wandbruntime‚3699  _runtime‚3699*%`WÒ'½ÌO NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59857129 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7952227747328 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚151072040111104 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 151397636 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58462979 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7743003616256 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 151387978 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58462977 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚92934657 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7952364712960 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚92925001 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7724052256952 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858139544232943e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚147318078132832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34647995 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚158815060766720 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7742916186624 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚151072057150464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚158814956297728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚151072040111104 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7742916186624 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58462977 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚155042130389784 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25209392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚701 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚92925001 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59857389 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0ÉN²~×'¾ÑO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚159380679922176 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚8799 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚151072057150464 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 151397636 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58462979 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7743003616256 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 151387978 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58462977 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚92934657 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7952364712960 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚92925001 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7724052256952 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 % _timestamp‚1.7858139544232943e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚147318078132832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34647995 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚158815060766720 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7742916186624 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚151072057150464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚158814956297728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚151072040111104 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7742916186624 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58462977 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚155042130389784 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25209392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚701 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚92925001 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59857389 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59857129 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7952227747328 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚151072040111104 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 151414816 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚147334818305152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151089223913472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858139547790642e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7724929730286 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 151414816 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7743795806208 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59864161 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚92935524 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58469622 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58469620 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ZFDeviceStatsMonit±“Úiˆor.on_train_batch_end/requested_bytes.large_pool.freed‚147334801265792 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚155059748035438 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚92935524 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34651934 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _step‚702 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25211975 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7743883243008 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚58469622 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7743795806208 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚158833107156480 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚151089206874112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34651932 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚151445684195328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59863907 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚151089223913472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚92945194 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚158833002680320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚3699.969322633 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7743883243008 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25212227 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚159398787907584 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚155059645192842 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚159398947672576 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚151445659291648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚8799â´gá&ÁÛM BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7743795806208 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59864161 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚92935524 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58469622 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58469620 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚147334801265792 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚155059748035438 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚92935524 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34651934 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 _step‚702 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25211975 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7743883243008 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚58469622 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7743795806208 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚252 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚158833107156480 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚151089206874112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34651932 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚151445684195328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59863907 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚151089223913472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚92945194 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚158833002680320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚3699.969322633 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7743883243008 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25212227 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚159398787907584 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚155059645192842 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚159398947672576 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚151445659291648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚8799 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 151405144 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58469620 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 151414816 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚147334818305152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151089223913472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858139547790642e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7724929730286 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 151414816 PDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚159717290583552 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚705 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60197885 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7997376284160 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚148155069709472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚159717395052544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60198145 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 152257788 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚93462659 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25353298 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚152288832918528 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25353040 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚93462659 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚151930395300864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34844845 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7786999751680 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7786912322048 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚148155086748832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58795127 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚151930395300864 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7786912322048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚155922924626808 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58795129 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚151930378261504 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34844847 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚160286350362624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7786999751680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _runtime‚3720.279633644 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚151930378261504 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚159717395052544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58795127 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858139750860837e+09à¥V4-ÎÑO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚151930378261504 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34844847 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.largeag(ª%_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚160286350362624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7786999751680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _runtime‚3720.279633644 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚151930378261504 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚159717395052544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58795127 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 % _timestamp‚1.7858139750860837e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚93453003 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7767940720460 trainer/global_step‚8849 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 152248130 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7767854917336 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚152288808014848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚159717290583552 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚159717290583552 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚705 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60197885 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7997376284160 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚148155069709472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚159717395052544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60198145 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 152257788 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚93462659 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25353298 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚152288832918528 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25353040 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚93462659 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚151930395300864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34844845 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7786999751680 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7786912322048 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚148155086748832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58795127 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚151930395300864 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7786912322048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚155922924626808 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58795129 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936Á%ÂlÄ2Ï- _wandbruntime‚3720  _runtime‚3720G$;‰Ü&ÐÖM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7768732390558 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7787879378432 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚151947562063872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7787879378432 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151947562063872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚3720.623133422 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7787791941632 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58801770 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60204644 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚152306040035328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚152306015131648 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7998411457024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚93463526 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34848782 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58801770 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚151947545024512 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7768818193794 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚93463526 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858139754328527e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚159735336966144 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7787791941632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚706 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 152274968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58801772 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 152274968 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34848784 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚160304295921664 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚93473196 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚151947545024512 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7768818193794 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚93463526 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858139754328527e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚159735336966144 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7787791941632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚706 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 152274968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58801772 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 152274968 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34848784 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚160304295921664 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚93473196 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151947562063872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚3720.623133422 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7787791941632 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58801770 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60204644 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚152306040035328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚152306015131648 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7998411457024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024ÂÚêŸ"2Ò- _wandbruntime‚3720  _runtime‚3720¦¨XÓÿ /loss/FRt1/assignment_loss‚2.048171621496959 0loss/FRt2/assignment_loss‚2.7952895098924637 2loss/FBt1/assignment_loss‚0.039903543914342934 3loss/FBt2/assignment_loss‚2.856797144594007e-05 .loss/FRt2/detection_loss‚2.374737151861191 1loss/SRqqt1/detection_loss‚0.9369005870434546 /loss/FBt2/detection_loss‚1.4773856626513104 2loss/SRqqt2/assignment_loss‚0.8264058677790066 epoch‚0 trainer/global_step‚8849 % _timestamp‚1.7858139754340656e+09 _runtime‚ 3720.62385482 /loss/FBt1/detection_loss‚1.4870427968225588 %loss/total_loss‚16.98753656675791 _step‚707 2loss/SRqqt1/assignment_loss‚0.9110226731612197 -loss/FRt1/detection_loss‚2.20801961133557 1loss/SRqqt2/detection_loss‚1.8826289728283883ÞÉ›x Ô„ -loss/FRt1/detection_loss‚2.20801961133557 1loss/SRqqt2/detection_loss‚1.8826289728283883 /loss/FRt1/assignment_loss‚2.048171621496959 0loss/FRt2/assignment_loss‚2.7952895098924637 2loss/FBt1/assignment_loss‚0.039903543914342934 3loss/FBt2/assignment_loss‚2.856797144594007e-05 .loss/FRt2/detection_loss‚2.374737151861191 1loss/SRqqt1/detection_loss‚0.9369005870434546 /loss/FBt2/detection_loss‚1.4773856626513104 2loss/SRqqt2/assignment_loss‚0.8264058677790066 epoch‚0 trainer/global_step‚8849 % _timestamp‚1.7858139754340656e+09 _runtime‚ 3720.62385482 /loss/FBt1/detection_loss‚1.4870427968225588 %loss/total_loss‚16.98753656675791 _step‚707 2loss/SRqqt1/assignment_loss‚0.9110226731612197Ã>æáœÕ‚(:‘ å·ÅÓ®áû‚ gpu.0.gpu‚39.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.829' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚32.276333333333326 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡¢gLKÖ‚(:Þ  å·ÅÓ÷¼Äƒ disk.drbd1828.in‚0 disk.drbd2118.out‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd2078.out‚0 disk.sda1.out‚0* proc.memory.percent‚17.172330617904663 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1817.in‚0$ disk./.usageGB‚589.3498573303223 disk.drbd1285.out‚0 disk.drbd1828.out‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.rbd0.in‚0) disk./.usagePercent‚32.95952923984553 disk.drbd2041.in‚0 disk.sda.out‚ 158.8984375 disk.drbd1226.in‚0" disk.dm-0.out‚3930.12060546875 disk.sda1.in‚0 disk.drbd1931.in‚0* proc.memory.availableMB‚38506.78515625 network.sent‚ 8.248595e+06 disk.drbd1552.in‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.dm-0.in‚ 431.34375 disk.drbd2118.in‚0 disk.drbd1289.out‚0 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.drbd2078.in‚0 proc.cpu.threads‚29 disk.drbd1817.out‚0 disk.sda2.out‚ 158.8984375 disk.sda.in‚11.625 disk.drbd2111.in‚0 cpu‚1»Ò½3.748687041959016 disk.drbd1552.out‚0 disk.drbd1938.out‚0 disk.sda2.in‚11.625 disk.drbd1226.out‚0 disk.drbd1965.in‚0% memory_percent‚41.243308782577515 disk.drbd1510.out‚0 disk.drbd2111.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1673.in‚0 disk.rbd1.out‚ 14.0224609375 disk.drbd1584.out‚0 disk.drbd1550.out‚0 network.recv‚560856 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚ 431.34375% disk.nvme0n1.out‚3930.12060546875ÖÚ3O2×- _wandbruntime‚3741  _runtime‚3741x>ÓÇØÁ _runtime‚3741.247831059 _step‚708 ' lr-AdamW/pg1‚0.00022934668753273895 ' lr-AdamW/pg2‚0.00022934668753273895 trainer/global_step‚8899 # _timestamp‚1.78581399605789e+09òçÌÙÆ _step‚708 ' lr-AdamW/pg1‚0.00022934668753273895 ' lr-AdamW/pg2‚0.00022934668753273895 trainer/global_step‚8899 # _timestamp‚1.78581399605789e+09 _runtime‚3741.247831059ÄT.12Ú- _wandbruntime‚3741  _runtime‚3741÷XW Ñ'ÛËO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25495567 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚153149188758528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7830995887104 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7830908457472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚264 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚156803821706316 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25495303 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚160619729338368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60537264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _step‚709 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚161191697741312 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7811829183968 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚152788733451264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚152788716411904 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 153117940 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59127279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚152788733451264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚160619624869376 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 $ _timestamp‚1.785813996058795e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚148992078325472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚160619624869376 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59127277 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59127277 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚153149163854848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _runtime‚3741.252624605 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚93981005 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7830908457472 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8042533886464 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚93990661 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚93990661 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35041697 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8042666657792 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7811743380844 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚8899 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59127279 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 153108282 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25495567 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚153149188758528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7830995887104 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7830908457472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚264 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚156803821706316 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25495303 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚160619729338368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60537264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _step‚709 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚161191697741312 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7811829183968 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚152788733451264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚152788716411904 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 153117940 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59127279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚152788733451264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚160619624869376 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 $ _timestamp‚1.785813996058795e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚148992078325472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚160619624869376 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59127277 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59127277 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚153149163854848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 _runtime‚3741.252624605 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚93981005 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7830908457472 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8042533886464 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚93990661 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚93990661 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35041697 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8042666657792 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7811743380844 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 trainer/global_step‚8899 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59127279 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 153108282 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0ůï+2Ý- _wandbruntime‚3741  _runtime‚37413üƒ Ü&ÞÖM VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25498197 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚161209811608576 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7812706657302 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59133920 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚152805883174912 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60543829 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35045632 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 153135120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚153166395875328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35045634 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚3741.593621361 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚94001198 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚149008818497792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 153125448 _step‚710 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7831788077056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59133920 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7831788077056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60544092 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8043440636928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 153135120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚152805900214272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚160637775728128 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚152805900214272 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7831875513856 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚152805883174912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8043569206784 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚156821439351858 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚160637775728128 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59133922 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7812620854066 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 153125448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚161209965082112 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚149008835537152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858139964033651e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚160637671251968 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚8899 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548.íÒðßÛM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60544092 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8043440636928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 153135120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚152805900214272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚160637775728128 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚152805900214272 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7831875513856 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚152805883174912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8043569206784 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚156821439351858 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚160637775728128 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59133922 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7812620854066 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 153125448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚161209965082112 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚149008835537152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858139964033651e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚160637671251968 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚8899 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25498197 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚161209811608576 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7812706657302 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59133920 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚152805883174912 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60543829 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35045632 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 153135120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚153166395875328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35045634 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚3741.593621361 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚94001198 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚149008818497792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 153125448 _step‚710 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7831788077056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59133920 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7831788077056Æëyè'2à- _wandbruntime‚3741  _runtime‚3741`Æ.á _step‚711 /loss/FRt1/assignment_loss‚2.276844225327174 2loss/SRqqt1/assignment_loss‚0.9023967804865208 .loss/FRt2/detection_loss‚2.684375136239188 epoch‚0 % _timestamp‚1.7858139964045868e+09 _runtime‚3741.594391664 2loss/SRqqt2/assignment_loss‚1.0781546803596227 2loss/FBt1/assignment_loss‚0.001571191710111649 /loss/FRt1/detection_loss‚2.2761492633985148 1loss/SRqqt1/detection_loss‚0.9982388135514512 /loss/FBt1/detection_loss‚1.8978557971346637 0loss/FRt2/assignment_loss‚2.7131562079702105 0loss/SRqqt2/detection_loss‚1.564892534247559 %loss/total_loss‚17.08559060174711 3loss/FBt2/assignment_loss‚0.0009130130167301033 /loss/FBt2/detection_loss‚0.6910429583053643 trainer/global_step‚8899D'\ ↠/loss/FRt1/detection_loss‚2.2761492633985148 1loss/SRqqt1/detection_loss‚0.9982388135514512 /loss/FBt1/detection_loss‚1.8978557971346637 0loss/FRt2/assignment_loss‚2.7131562079702105 0loss/SRqqt2/detection_loss‚1.564892534247559 %loss/total_loss‚17.08559060174711 3loss/FBt2/assignment_loss‚0.0009130130167301033 /loss/FBt2/detection_loss‚0.6910429583053643 trainer/global_step‚8899 _step‚711 /loss/FRt1/assignment_loss‚2.276844225327174 2loss/SRqqt1/assignment_loss‚0.9023967804865208 .loss/FRt2/detection_loss‚2.684375136239188 epoch‚0 % _timestamp‚1.7858139964045868e+09 _runtime‚3741.594391664 2loss/SRqqt2/assignment_loss‚1.0781546803596227 2loss/FBt1/assignment_loss‚0.001571191710111649ǵ›ƒœã‚(:‘ ô·ÅÓ™„ƒ gpu.0.gpu‚39.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚128.736' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.91199999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0˜ÓÊ•jä‚(:ß  ô·ÅÓ¿ßŃ$ proc.memory.rssMB‚11254.05859375 cpu‚13.748134827344838# disk./.usageGB‚589.347354888916 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.drbd2041.out‚0 disk.sda.in‚11.625 disk.drbd2111.out‚0 disk.drbd2041.in‚0 disk.drbd1226.out‚0 disk.drbd1965.out‚0 disk.drbd1510.in‚0% memory_percent‚41.243404150009155 network.sent‚ 8.293619e+06% disk.nvme0n1.out‚3930.69873046875 disk.drbd1965.in‚0* proc.memory.availableMB‚38506.72265625 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1550.in‚0 disk.drbd1828.out‚0 disk.sda.out‚ 159.68359375 disk.drbd1285.out‚0 disk.drbd2118.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.drbd1828.in‚0 disk.dm-0.in‚ 431.34375 disk.drbd1289.in‚0 disk.sda1.in‚0 disk.drbd1673.out‚0 disk.sda2.out‚ 159.68359375 disk.rbd0.in‚0 disk.sda1.out‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚11.625 disk.rbd1.out‚ 14.0224609375 disk.drbd1356.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0" disk.dm-0.out‚3930.69873046875 disk.drbd1356.out‚0 disk.drbd1510.out‚0 proc.cpu.threads‚29 disk.nvme0n1.in‚ 431.34375 disk.drbd1285.in‚0 disk.drbd1552.out‚0) disk./.usagePercent‚32.95938929022193 disk.drbd1817.in‚0 disk.drbd1552.in‚0 network.recv‚562641 disk.drbd1226.in‚0 disk.drbd2118.in‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.drbd2111.in‚0øŸ2å- _wandbruntime‚3762  _runtime‚3762^{ˆœÆæÀ trainer/global_step‚8949 % _timestamp‚1.7858140169254763e+09 _runtime‚ 3762.11539903 _step‚712 & lr-AdamW/pg1‚0.0002306352968570043 & lr-AdamW/pg2‚0.0002306352968570043KuVËçÅ & lr-AdamW/pg1‚0.0002306352968570043 & lr-AdamW/pg2‚0.0002306352968570043 trainer/global_step‚8949 % _timestamp‚1.7858140169254763e+09 _runtime‚ 3762.11539903 _step‚712Èzk]f2è- _wandbruntime‚3762  _runtime‚3762ÈÒ'éÌO BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60877363 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚157684718785824 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚161522063624192 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8087871456256 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚161521959155200 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7855717647476 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚153647054562304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚153647071601664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35238545 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59459429 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚94518663 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚713 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚161522063624192 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60877625 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚154009519694848 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚153647071601664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚154009544598528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚149829086941472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7874904592896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7874992022528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7874992022528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚162097258379776 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59459427 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35238547 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59459429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7874904592896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚162097416054784 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 153978092 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25639078 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59459427 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚8949 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚94509007 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚161521959155200 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚153647054562304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25638818 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858140169263828e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 153968434 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚94518663 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60877363 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚157684718785824 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚161522063624192 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8087871456256 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚161521959155200 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7855717647476 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚153647054562304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚153647071601664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35238545 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59459429 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚94518663 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _step‚713 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚161522063624192 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60877625 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚154009519694848 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚153647071601664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚154009544598528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚149829086941472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7874904592896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7874992022528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7874992022528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚162097258379776 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59459427 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35238547 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59459429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7874904592896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚162097416054784 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 153978092 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25639078 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59459427 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚8949 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚94509007 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚161521959155200 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚153647054562304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inact¼¨"[.ive_split.small_pool.freed‚25638818 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858140169263828e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 153968434 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚94518663 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0ÉÄ,2ë- _wandbruntime‚3762  _runtime‚3762·c4Ú&ìÔM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7875871649280 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚157702439273962 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 153995272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚714 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚153664238364672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚153664221325312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚157702336431366 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60884170 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚153664221325312 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚3762.470626238 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚94529200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚161540110013952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 153985600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚162115370297344 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚161540110013952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8088774152704 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35242484 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚161540005537792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚94519530 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚149845827113792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60884436 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 # _timestamp‚1.78581401728031e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59466070 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25641688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚154026726811648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7856509317574 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 153985600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚8949 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7875871649280 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25641952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35242482 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7875784212480 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7875784212480 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚162115525868032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚149845844153152Ò²Lïß&íÙM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚153664221325312 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚3762.470626238 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚94529200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚161540110013952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 153985600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚162115370297344 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚161540110013952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8088774152704 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35242484 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚161540005537792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚94519530 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚149845827113792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60884436 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 # _timestamp‚1.78581401728031e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59466070 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25641688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚154026726811648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7856509317574 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 153985600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚8949 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7875871649280 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25641952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35242482 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7875784212480 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7875784212480 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚162115525868032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚149845844153152 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7875871649280 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚157702439273962 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 153995272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚714 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚153664238364672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚153664221325312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚157702336431366 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60884170 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032Ê’…oó2î- _wandbruntime‚3762  _runtime‚3762O[ ï† 1loss/SRqqt1/detection_loss‚0.9771038587117251 &loss/total_loss‚17.314060683702756 % _timestamp‚1.7858140172815664e+09 epoch‚0 2loss/SRqqt1/assignment_loss‚1.0080598115470003 3loss/FBt2/assignment_loss‚7.705975162102569e-07 /loss/FRt1/detection_loss‚2.3808615989173236 /loss/FBt2/detection_loss‚0.5732495706262333 1loss/SRqqt2/detection_loss‚1.7919281097128987 /loss/FBt1/detection_loss‚1.3741177237592637 trainer/global_step‚8949 _runtime‚3762.471349552 _step‚715 /loss/FRt1/assignment_loss‚2.433086082339287 0loss/FRt2/assignment_loss‚2.7636169435141924 2loss/SRqqt2/assignment_loss‚1.3782234202080872 4loss/FBt1/assignment_loss‚2.8469451447321605e-05 /loss/FRt2/detection_loss‚2.6337843243177836ßAð‹ _step‚715 /loss/FRt1/assignment_loss‚2.433086082339287 0loss/FRt2/assignment_loss‚2.7636169435141924 2loss/SRqqt2/assignment_loss‚1.3782234202080872 4loss/FBt1/assignment_loss‚2.8469451447321605e-05 /loss/FRt2/detection_loss‚2.6337843243177836 1loss/SRqqt1/detection_loss‚0.9771038587117251 &loss/total_loss‚17.314060683702756 % _timestamp‚1.7858140172815664e+09 epoch‚0 2loss/SRqqt1/assignment_loss‚1.0080598115470003 3loss/FBt2/assignment_loss‚7.705975162102569e-07 /loss/FRt1/detection_loss‚2.3808615989173236 /loss/FBt2/detection_loss‚0.5732495706262333 1loss/SRqqt2/detection_loss‚1.7919281097128987 /loss/FBt1/detection_loss‚1.3741177237592637 trainer/global_step‚8949 _runtime‚3762.471349552ËýC€ñ‚(:’ ƒ¸ÅÓÒò‚ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.156' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.385333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦xé¡aò‚(:Ö  ƒ¸ÅÓÝÕ¸ƒ disk.drbd1226.in‚0 disk.dm-0.in‚ 431.34375 disk.drbd1938.out‚0 disk.drbd1510.out‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.drbd1673.in‚0 disk.sda2.in‚11.625 disk.rbd1.out‚ 14.0224609375 disk.drbd2041.out‚0$ memory_percent‚41.24383330345154 proc.cpu.threads‚29 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.drbd1828.in‚0 disk.drbd2118.in‚0 disk.drbd1817.out‚0 disk.drbd1584.out‚0 disk.drbd1356.out‚0 disk.drbd1285.out‚0 disk.drbd1828.out‚0 network.recv‚564167 disk.drbd1550.out‚0 disk.sda2.out‚ 160.28125 cpu‚13.74794698178746 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.sda1.in‚0 disk.nvme0n1.in‚ 431.34375$ disk.nvme0n1.out‚3931.2119140625 disk.drbd1931.out‚0 disk.drbd1510.in‚0 disk.sda.out‚ 160.28125 disk.drbd1550.in‚0 disk.drbd1817.in‚0 disk.drbd2111.out‚0 disk.drbd1965.out‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.drbd1584.in‚0 disk.sda.in‚11.625 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.drbd1552.in‚0 disk.rbd1.in‚0* proc.memory.availableMB‚38506.44140625* proc.memory.percent‚17.172330617904663 disk.drbd1931.in‚0 disk.drbd1289.in‚0$ disk./.usageGB‚589.3499794006348! disk.dm-0.out‚3931.2119140625) disk./.usagePercent‚32.95953606665643 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd1356.in‚0 network.sent‚ 8.338674e+06 disk.drbd2078.in‚0$ proc.memory.rssMB‚11254.05859375Ëm¶&›ó‚(: ’¸ÅÓÑ­ƒ gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚97.355' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.45166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0½÷öwgô‚(:Ü  ’¸ÅÓúÍ⃠disk.rbd1.out‚ 14.0224609375* proc.memory.percent‚17.172330617904663 network.recv‚565717 disk.drbd1550.out‚0 disk.dm-0.in‚ 431.34375 disk.drbd1510.out‚0* proc.memory.availableMB‚38506.68359375 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd1828.out‚0 disk.drbd1552.out‚0 disk.rbd1.in‚0$ disk.nvme0n1.out‚3931.5048828125 disk.drbd2111.out‚0 disk.sda.out‚ 160.66015625! disk.dm-0.out‚3931.5048828125 network.sent‚ 8.383853e+06 disk.drbd1285.out‚0 disk.drbd1938.in‚0 disk.drbd1356.out‚0# disk./.usageGB‚589.350902557373 disk.drbd1673.in‚0 disk.drbd1673.out‚0 proc.cpu.threads‚29 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.sda.in‚11.625 disk.drbd1510.in‚0 disk.drbd2118.out‚0$ memory_percent‚41.24346375465393) disk./.usagePercent‚32.95958769441392 disk.drbd1965.out‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.drbd1931.out‚0 disk.drbd2078.in‚0 disk.drbd2118.in‚0 disk.sda2.in‚11.625 disk.sda2.out‚ 160.66015625 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.drbd1817.out‚0 disk.nvme0n1.in‚ 431.34375 cpu‚13.747790001022397 disk.sda1.in‚0 disk.drbd1584.in‚0 disk.drbd1552.in‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1285.in‚0 disk.drbd1938.out‚0 disk.drbd1584.out‚0 disk.drbd1226.out‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1289.in‚0»bâÍ2õ- _wandbruntime‚3782  _runtime‚3782.õë­Èö _step‚716 ' lr-AdamW/pg1‚0.00023192390618126957 ' lr-AdamW/pg2‚0.00023192390618126957 trainer/global_step‚8999 $ _timestamp‚1.785814037750009e+09 _runtime‚3782.939992661EŒb˜Í÷Ç _step‚716 ' lr-AdamW/pg1‚0.00023192390618126957 ' lr-AdamW/pg2‚0.00023192390618126957 trainer/global_step‚8999 $ _timestamp‚1.785814037750009e+09 _runtime‚3782.939992661Ì9–®³2ø- _wandbruntime‚3782  _runtime‚3782ú–/ùÌO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59791579 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7918988157952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚717 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 trainer/global_step‚8999 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 154838244 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚162424293441024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59791577 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚154505392712704 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚154505409752064 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚154505409752064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61216521 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚163002801791488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35435395 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59791577 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8133056930816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚162424397910016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59791579 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25781360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚95046665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚162424293441024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚150666112596832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 154828586 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7918988157952 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚95046665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚154869900438528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8132926256640 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚158565718707816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚95037009 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61216757 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚154505392712704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 VBDeviceStatsMonitor.on_train_batêþ.>£ch_start/active_bytes.all.allocated‚162424397910016 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚150666095557472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚163002957369344 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚95037009 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚158565615865332 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7899606110984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7918900728320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚154869875534848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7918900728320 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚163002957369344 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚95037009 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚158565615865332 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7899606110984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7918900728320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚154869875534848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7918900728320 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚162424293441024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59791577 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 130674176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚154505392712704 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚154505409752064 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚154505409752064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61216521 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚163002801791488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35435395 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59791577 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8133056930816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚162424397910016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59791579 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25781360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚95046665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚162424293441024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚150666112596832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 154828586 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7918988157952 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚95046665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚154869900438528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8132926256640 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚158565718707816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚95037009 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61216757ÍêWë2û- _wandbruntime‚3783  _runtime‚3783NÓÓÛ&üÕM H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 trainer/global_step‚8999 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59798222 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚95057202 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59798222 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚163021069976064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 154855424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚158583233510874 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8133825462272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚150682835729792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚150682852769152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚163020908113920 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚154887082651648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59798220 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚154522559475712 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚95047532 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚158583336353470 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚95057202 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154522576515072 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154522559475712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 154845752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7900483584318 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59798220 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7900397781082 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7919867784704 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25784198 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61223275 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 154845752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7919780347904 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚95047532 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7919780347904 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136ªAýV ýÚM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 154845752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7919780347904 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚95047532 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7919780347904 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 trainer/global_step‚8999 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59798222 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚95057202 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59798222 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚163021069976064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 154855424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚158583233510874 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8133825462272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚150682835729792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚150682852769152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚163020908113920 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚154887082651648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59798220 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚154522559475712 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚95047532 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚255 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚158583336353470 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚95057202 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154522576515072 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154522559475712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 154845752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7900483584318 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes}Ä Š.all.current‚ 104476160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59798220 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7900397781082 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7919867784704 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25784198 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61223275Îsü42þ- _wandbruntime‚3783  _runtime‚3783FVÚ'ÿÿ /loss/FRt1/detection_loss‚2.3767490977986476 0loss/FRt2/assignment_loss‚2.4629712368868573 .loss/FRt2/detection_loss‚2.734573543071747 1loss/SRqqt2/detection_loss‚1.8983552327845246 .loss/FBt1/detection_loss‚1.395414037629962 %loss/total_loss‚16.51460192213205 trainer/global_step‚8999 _step‚719 1loss/SRqqt1/detection_loss‚0.9433672200561828 /loss/FBt2/detection_loss‚1.0185283363539854 epoch‚0 3loss/FBt2/assignment_loss‚6.235412601148678e-06 $ _timestamp‚1.785814038105068e+09 _runtime‚3783.294848073 /loss/FRt1/assignment_loss‚1.973575118552433 1loss/SRqqt1/assignment_loss‚0.908963641405027 2loss/SRqqt2/assignment_loss‚0.7854723704745993 2loss/FBt1/assignment_loss‚0.016625851705482222ªÁOW €„ /loss/FBt2/detection_loss‚1.0185283363539854 epoch‚0 3loss/FBt2/assignment_loss‚6.235412601148678e-06 $ _timestamp‚1.785814038105068e+09 _runtime‚3783.294848073 /loss/FRt1/assignment_loss‚1.973575118552433 1loss/SRqqt1/assignment_loss‚0.908963641405027 2loss/SRqqt2/assignment_loss‚0.7854723704745993 2loss/FBt1/assignment_loss‚0.016625851705482222 /loss/FRt1/detection_loss‚2.3767490977986476 0loss/FRt2/assignment_loss‚2.4629712368868573 .loss/FRt2/detection_loss‚2.734573543071747 1loss/SRqqt2/detection_loss‚1.8983552327845246 .loss/FBt1/detection_loss‚1.395414037629962 %loss/total_loss‚16.51460192213205 trainer/global_step‚8999 _step‚719 1loss/SRqqt1/detection_loss‚0.9433672200561828Ï©º_œ‚(:‘ ¡¸ÅÓ–Ÿƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚128.297' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.76566666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0VÌÁi‚‚(:Þ  ¡¸ÅÓÑßµƒ disk.drbd2078.in‚0" disk.dm-0.out‚3932.48291015625 disk.drbd1510.out‚0 disk.drbd1584.out‚0 disk.drbd2078.out‚0 disk.drbd1817.in‚0 network.sent‚ 8.387031e+06 disk.drbd1226.out‚0 disk.drbd1550.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1931.out‚0 disk.drbd2041.out‚0 disk.sda.in‚11.625 disk.sda.out‚ 161.13671875 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.sda2.in‚11.625 cpu‚13.747347528287436 disk.rbd0.in‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.drbd1289.in‚0 disk.drbd1673.out‚0 disk.drbd2111.in‚0 disk.drbd1938.in‚0 disk.drbd1931.in‚0 disk.nvme0n1.in‚ 431.34375 disk.drbd2041.in‚0 disk.drbd1226.in‚0 disk.sda2.out‚ 161.13671875 disk.drbd1356.in‚0 disk.drbd1965.out‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0* proc.memory.availableMB‚38506.59765625( disk./.usagePercent‚32.9595844943463$ proc.memory.rssMB‚11254.05859375 disk.drbd1289.out‚0% disk.nvme0n1.out‚3932.48291015625 disk.dm-0.in‚ 431.34375 disk.drbd1938.out‚0$ disk./.usageGB‚589.3508453369141 disk.drbd1584.in‚0 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2118.out‚0$ memory_percent‚41.24359488487244 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.rbd1.out‚ 14.0224609375 disk.rbd1.in‚0 proc.cpu.threads‚29 network.recv‚566740 disk.drbd1356.out‚0O2ƒ- _wandbruntime‚3804  _runtime‚38048ŠßÉ„Ã _runtime‚3804.043304356 _step‚720 ' lr-AdamW/pg1‚0.00023321251550553488 ' lr-AdamW/pg2‚0.00023321251550553488 trainer/global_step‚9049 % _timestamp‚1.7858140588533766e+09«E¼)Î…È ' lr-AdamW/pg1‚0.00023321251550553488 ' lr-AdamW/pg2‚0.00023321251550553488 trainer/global_step‚9049 % _timestamp‚1.7858140588533766e+09 _runtime‚3804.043304356 _step‚720к2†- _wandbruntime‚3804  _runtime‚3804N}TàÒ'‡ÌO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚163326627726848 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60123727 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8178234053120 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚95574667 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚232 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚155363730863104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60123729 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚155363747902464 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 % _timestamp‚1.7858140588542817e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚159446512944840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7943408771368 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚163326627726848 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚230 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚163326732195840 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7962984293376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7962896863744 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7943494574492 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚155363730863104 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35632245 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚163908332656640 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 155688738 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60123727 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚95565011 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚151503121212832 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 155688738 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60123727 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚95565011 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚151503121212832 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚95574667 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚232 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚155363730863104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60123729 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚155363747902464 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 % _timestamp‚1.7858140588542817e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚159446512944840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7943408771368 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚163326627726848 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚230 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚163326732195840 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7962984293376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7962896863744 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7943494574492 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚155363730863104 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35632245 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚163908332656640 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 155705904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60130372 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 155715576 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35636184 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚159464130590382 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚151519861385152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚95575534 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60130370 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 155705904 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7944372047826 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚95585204 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚155747438491648 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _runtime‚3804.398185466 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚155380897626112 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚95575534 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚155380914665472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚163344778585600 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60130372 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚275 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7944286244590 _step‚722 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚155747463395328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7963776483328 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚277 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60130370 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7963863920128 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚163344674109440 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚95585204 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61563738 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25927279 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚ê9Œtã 0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7963776483328 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚155380914665472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61563461 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚159464233432978 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8179012734464 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 155715576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚163344778585600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚155380897626112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 $ _timestamp‚1.785814059207916e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8179149692928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 155705904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60130372 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 155715576 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35636184 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚159464130590382 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚151519861385152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚95575534 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60130370 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 155705904 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7944372047826 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚95585204 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚155747438491648 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _runtime‚3804.398185466 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚155380897626112 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚95575534 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚155380914665472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚163344778585600 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60130372 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚275 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7944286244590 _step‚722 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚155747463395328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7963776483328 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚277 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60130370 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7963863920128 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚163344674109440 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚95585204 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61563738 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25927279 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7963776483328 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚155380914665472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61563461 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚159464233432978 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8179012734464 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 155715576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚163344778585600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚155380897626112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 $ _timestamp‚1.785814059207916e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8179149692928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚160327512866832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7987383038000 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚156590612118528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164228962012672 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚164813894850048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164228962012672 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7858140793520708e+09 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚96093013 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35829097 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚156590587214848 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚156222086052864 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8006892999168 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚152340112789472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚96102669C™‰Ú–ÐO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚156222069013504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8223436212224 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7987297234876 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 156548890 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚96093013 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60455877 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164228962012672 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚164813894850048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164228962012672 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7858140793520708e+09 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚96093013 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35829097 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚156590587214848 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚156222086052864 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8006892999168 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚152340112789472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚96102669 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61897333 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚156222086052864 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8006980428800 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60455879 _runtime‚ 3824.54567115 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚160327512866832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7987383038000 VGDeviceStatsÐO}üMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚156590612118528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35833034 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8224207332352 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 156566056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61904141 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚152356870001152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 $ _timestamp‚1.785814079702382e+09 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 156575728 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚9099 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26071109 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚96103536 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚164247112871424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚156239252815872 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7988260511334 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60462522 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8007772618752 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚160345130512486 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60462520 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚156239252815872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164247008395264 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7988174708098 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8007860055552 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8224346387968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60462520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61904400 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚96113206 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚156239235776512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60462522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚164247112871424 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 156575728 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚156239235776512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _runtime‚3824.892655075¾ífà&™ÚM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚156239235776512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _runtime‚3824.892655075 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8007860055552 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 156566056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35833034 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8224207332352 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 156566056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61904141 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚152356870001152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 $ _timestamp‚1.785814079702382e+09 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 156575728 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚9099 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26071109 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚96103536 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚164247112871424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚156239252815872 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7988260511334 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60462522 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8007772618752 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚160345130512486 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60462520 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚156239252815872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164247008395264 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7988174708098 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8007860055552 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 139055616 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8224346387968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60462520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61904400 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚96113206 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚156239235776512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60462522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚164247112871424 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 156575728 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888Ö`ã/2š- _wandbruntime‚3824  _runtime‚3824³rµ¦ ›ƒ trainer/global_step‚9099 % _timestamp‚1.7858140797036054e+09 0loss/FRt1/assignment_loss‚2.3600470200181007 3loss/FBt2/assignment_loss‚5.781623578116068e-07 _runtime‚3824.893378232 _step‚727 2loss/SRqqt2/assignment_loss‚1.0158605506966034 /loss/FRt1/detection_loss‚2.3260830086655915 /loss/FRt2/detection_loss‚2.9349624203212223 0loss/SRqqt2/detection_loss‚1.562087728952368 /loss/FBt2/detection_loss‚1.3266614305786788 3loss/FBt1/assignment_loss‚6.939627829589361e-06 %loss/total_loss‚17.23688917968227 epoch‚0 0loss/FRt2/assignment_loss‚2.6486897148303132 1loss/SRqqt1/assignment_loss‚0.859858742596319 1loss/SRqqt1/detection_loss‚0.8677253668265255 /loss/FBt1/detection_loss‚1.3349056784063578%L“Øœˆ /loss/FBt2/detection_loss‚1.3266614305786788 3loss/FBt1/assignment_loss‚6.939627829589361e-06 %loss/total_loss‚17.23688917968227 epoch‚0 0loss/FRt2/assignment_loss‚2.6486897148303132 1loss/SRqqt1/assignment_loss‚0.859858742596319 1loss/SRqqt1/detection_loss‚0.8677253668265255 /loss/FBt1/detection_loss‚1.3349056784063578 trainer/global_step‚9099 % _timestamp‚1.7858140797036054e+09 0loss/FRt1/assignment_loss‚2.3600470200181007 3loss/FBt2/assignment_loss‚5.781623578116068e-07 _runtime‚3824.893378232 _step‚727 2loss/SRqqt2/assignment_loss‚1.0158605506966034 /loss/FRt1/detection_loss‚2.3260830086655915 /loss/FRt2/detection_loss‚2.9349624203212223 0loss/SRqqt2/detection_loss‚1.562087728952368×P‰à5œ‚(:‘ ¿¸ÅÓ–ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚101.194' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚33.73133333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÎôQÉež‚(:Ú  ¿¸ÅÓ³ÔÛƒ disk.drbd1584.in‚0 cpu‚13.7468192567973 disk.drbd1965.in‚0 disk.drbd1673.out‚0 disk.sda2.out‚ 162.1484375 disk.drbd1356.in‚0 disk.rbd1.out‚ 14.0224609375 disk.drbd1550.out‚0 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.drbd1552.out‚0 disk.drbd1817.out‚0 proc.cpu.threads‚29* disk./.usagePercent‚32.959584067670626 disk.drbd1510.in‚0 disk.drbd1938.out‚0 disk.drbd2041.out‚0 disk.sda.in‚11.625 disk.drbd2111.in‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚ 431.34375 disk.drbd1226.in‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0! disk.dm-0.out‚3933.0009765625 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.drbd1931.out‚0 disk.sda.out‚ 162.1484375 disk.rbd1.in‚0* proc.memory.percent‚17.172330617904663 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.rbd0.out‚0* proc.memory.availableMB‚38506.26953125 disk.drbd1584.out‚0 disk.sda1.out‚0 disk.rbd0.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd1285.out‚0 disk.drbd1938.in‚0 disk.sda1.in‚0 disk.drbd2118.in‚0$ disk.nvme0n1.out‚3933.0009765625 disk.drbd1965.out‚0 disk.sda2.in‚11.625 disk.drbd1356.out‚0 disk.dm-0.in‚ 431.34375 network.sent‚ 8.477123e+06$ disk./.usageGB‚589.3508377075195 disk.drbd2078.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.drbd1931.in‚0$ proc.memory.rssMB‚11254.05859375 network.recv‚569882 disk.drbd2041.in‚0$ memory_percent‚41.24409556388855¯üŸ‚(:… θÅÓÏ߃ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚113.097' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.699 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0j eˆi ‚(:Þ  θÅÓˆúæƒ disk.drbd1965.out‚0 disk.drbd1673.out‚0 disk.rbd1.out‚ 14.0224609375* proc.memory.availableMB‚38505.91015625 disk.drbd1285.out‚0 disk.drbd2118.in‚0 network.recv‚571218$ disk./.usageGB‚589.3508567810059 disk.drbd1584.out‚0 disk.drbd1226.out‚0 disk.dm-0.in‚ 431.34375 disk.drbd1356.in‚0 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.drbd1552.in‚0 cpu‚13.746478542029163 disk.drbd1356.out‚0 disk.sda2.out‚ 162.57421875$ disk.nvme0n1.out‚3933.2236328125 disk.rbd0.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚11.625 disk.drbd1828.in‚0! disk.dm-0.out‚3933.2236328125 disk.rbd1.in‚0 disk.drbd1931.out‚0 disk.sda.in‚11.625 disk.drbd2078.out‚0 disk.sda1.in‚0 disk.drbd1817.out‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.sda1.out‚0$ memory_percent‚41.24464392662048 network.sent‚ 8.522083e+06 disk.sda.out‚ 162.57421875 disk.drbd2041.out‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2111.in‚0 disk.drbd1938.in‚0* disk./.usagePercent‚32.959585134359834 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.drbd1289.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd1510.out‚0 disk.nvme0n1.in‚ 431.34375 disk.drbd1584.in‚0 disk.drbd2041.in‚0 disk.drbd1552.out‚0 disk.drbd1673.in‚0 proc.cpu.threads‚29.±+2¡- _wandbruntime‚3845  _runtime‚3845:=%ìȢ ' lr-AdamW/pg1‚0.00023578973415406548 ' lr-AdamW/pg2‚0.00023578973415406548 trainer/global_step‚9149 % _timestamp‚1.7858141003236926e+09 _runtime‚ 3845.51561805 _step‚728µiâꣂ ' lr-AdamW/pg1‚0.00023578973415406548 ' lr-AdamW/pg2‚0.00023578973415406548 trainer/global_step‚9149 % _timestamp‚1.7858141003236926e+09 _runtime‚ 3845.51561805 _step‚728ز(€Û2¤- _wandbruntime‚3845  _runtime‚3845ð{Yk¥ÌO BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚165131296298496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚165131296298496 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26210754 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 157418700 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8050976564224 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚9149 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚161208409946340 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8050976564224 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36025945 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3845.515701306 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8050889134592 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚157080407163904 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚157080424203264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚157080407163904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚153177138444832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60788027 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚165131400767488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8031185698384 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 157409042uÒÅØ×'¦ÑO PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚96630671 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8050889134592 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚60788029 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚157450967958528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚153177121405472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚161208307103856 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚165131400767488 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8050889134592 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚157080407163904 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚157080424203264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚157080407163904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚153177138444832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60788027 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚165131400767488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8031185698384 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 157409042 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚165131296298496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚165131296298496 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26210754 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 157418700 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8050976564224 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 trainer/global_step‚9149 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚161208409946340 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8050976564224 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36025945 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3845.515701306 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 157435880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚165149447157248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60794672 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚157468175075328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 % _timestamp‚1.7858141006660802e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60794672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8032063171606 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚96641208 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚157468150171648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _step‚730 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚165149447157248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8269544605696 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚165737562013184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8051856190976 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62243724 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 157426208 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8051856190976 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60794670 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚157097573926912 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚157097590966272 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8269411841536 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚236 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚161225924749398 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 157435880 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8051768754176 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62243488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 157426208 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36029882 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚165737719681024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 trainer/global_step‚9149 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚96631538 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26213840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚153193878617152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚96631538 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60794670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚153193861577792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚157097590966272 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8051768754176 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚165149342681088 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36029884 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚161226027591994 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160îþÛ7‘ ©ÛM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8051856190976 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62243724 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 157426208 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8051856190976 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60794670 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚157097573926912 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚157097590966272 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8269411841536 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚236 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚161225924749398 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 157435880 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8051768754176 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62243488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 157426208 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36029882 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚165737719681024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 trainer/global_step‚9149 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚96631538 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26213840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚153193878617152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚96631538 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60794670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚153193861577792 O@DeviceStat˜9ÀPsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚157097590966272 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8051768754176 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚165149342681088 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36029884 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚161226027591994 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚96641208 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 157435880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚165149447157248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60794672 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚157468175075328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 % _timestamp‚1.7858141006660802e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60794672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8032063171606 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚96641208 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚157468150171648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _step‚730 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚165149447157248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8269544605696 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚165737562013184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121ÚD«Uï2ª- _wandbruntime‚3845  _runtime‚3845$°÷ø «ƒ 2loss/FBt2/assignment_loss‚0.025359868780911704 1loss/SRqqt1/detection_loss‚0.9826119663445531 /loss/FBt2/detection_loss‚0.6244405772344306 _runtime‚3845.857057952 _step‚731 0loss/FRt1/assignment_loss‚2.2638568661429663 1loss/SRqqt2/detection_loss‚1.7252424084284204 /loss/FBt1/detection_loss‚1.6852166184357233 epoch‚0 0loss/FRt2/assignment_loss‚2.4212574972503487 2loss/SRqqt1/assignment_loss‚0.9468869307059543 2loss/SRqqt2/assignment_loss‚1.0764850657532348 2loss/FBt1/assignment_loss‚0.000264716437811785 &loss/total_loss‚16.524895105744022 trainer/global_step‚9149 /loss/FRt1/detection_loss‚2.2563839926563127 /loss/FRt2/detection_loss‚2.5168885975733573 $ _timestamp‚1.785814100667274e+09Ø𬈠0loss/FRt2/assignment_loss‚2.4212574972503487 2loss/SRqqt1/assignment_loss‚0.9468869307059543 2loss/SRqqt2/assignment_loss‚1.0764850657532348 2loss/FBt1/assignment_loss‚0.000264716437811785 &loss/total_loss‚16.524895105744022 trainer/global_step‚9149 /loss/FRt1/detection_loss‚2.2563839926563127 /loss/FRt2/detection_loss‚2.5168885975733573 $ _timestamp‚1.785814100667274e+09 2loss/FBt2/assignment_loss‚0.025359868780911704 1loss/SRqqt1/detection_loss‚0.9826119663445531 /loss/FBt2/detection_loss‚0.6244405772344306 _runtime‚3845.857057952 _step‚731 0loss/FRt1/assignment_loss‚2.2638568661429663 1loss/SRqqt2/detection_loss‚1.7252424084284204 /loss/FBt1/detection_loss‚1.6852166184357233 epoch‚0Û0[x˜œ­‚(:‘ ݸÅÓ»îñ‚ gpu.0.gpu‚29.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.481' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.49366666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0žZ®‚(:Ï  ݸÅÓ§„ºƒ disk.drbd1965.in‚0 disk.sda1.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 disk.sda.in‚11.625 disk.drbd1550.in‚0 disk.drbd1226.out‚0 disk.drbd2118.out‚0 disk.rbd0.out‚0* proc.memory.percent‚17.172330617904663) disk./.usagePercent‚32.95956017383245 disk.drbd1552.in‚0 network.sent‚ 8.525109e+06 disk.drbd2078.out‚0 disk.drbd1828.out‚0 disk.dm-0.out‚3933.603515625% memory_percent‚41.245222091674805 proc.cpu.threads‚29 disk.drbd1584.in‚0 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.sda1.in‚0 cpu‚13.746335082762512 disk.rbd1.out‚ 14.0224609375 disk.drbd1550.out‚0 disk.drbd1285.out‚0 disk.drbd1356.in‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.nvme0n1.in‚ 431.34375 disk.drbd1931.in‚0 disk.drbd2111.in‚0 network.recv‚572209$ disk./.usageGB‚589.3504104614258 disk.drbd1938.out‚0 disk.sda2.in‚11.625 disk.drbd1584.out‚0 disk.drbd1226.in‚0' proc.memory.availableMB‚ 38505.53125 disk.drbd1817.out‚0 disk.rbd1.in‚0 disk.sda.out‚163.375 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.sda2.out‚163.375 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.dm-0.in‚ 431.34375 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.drbd2041.out‚0 disk.drbd1552.out‚0# disk.nvme0n1.out‚3933.603515625 disk.drbd2041.in‚0 disk.drbd1965.out‚0 disk.drbd1673.in‚0'Q æ2¯- _wandbruntime‚3865  _runtime‚3865ËÙµJÆ°À & lr-AdamW/pg1‚0.0002370783434783308 & lr-AdamW/pg2‚0.0002370783434783308 trainer/global_step‚9199 $ _timestamp‚1.785814120807454e+09 _runtime‚3865.997386668 _step‚732¡o˱Š_runtime‚3865.997386668 _step‚732 & lr-AdamW/pg1‚0.0002370783434783308 & lr-AdamW/pg2‚0.0002370783434783308 trainer/global_step‚9199 $ _timestamp‚1.785814120807454e+09ܨe_62²- _wandbruntime‚3866  _runtime‚3866Í—Ñ'³ËO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8094972699648 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚97158673 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚166033630584320 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚166624980545536 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚97158673 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62576446 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚166033735053312 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.785814120808363e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚97149017 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚158311298894848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _step‚733 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61120177 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162089307025848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚166033630584320 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚61120177 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 158269194 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚166625138220544 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚154014130021472 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚157938762353664 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36222795 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26353909 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8094885270016 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚158311323798528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8313814422016 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26353651 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8313681650688 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚157938745314304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚97149017 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8075074161892 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 158278852 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚97158673 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚166033630584320 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚166624980545536 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚97158673 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62576446 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚166033735053312 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.785814120808363e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚97149017 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚158311298894848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 _step‚733 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61120177 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162089307025848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚166033630584320 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚61120177 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 158269194 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚166625138220544 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚154014130021472 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚157938762353664 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36222795 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26353909 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8094885270016 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚158311323798528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8313814422016 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26353651 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8313681650688 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚157938745314304ÝS¤5,2µ- _wandbruntime‚3866  _runtime‚3866)<éA¶ÖM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚97169210 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26356711 % _timestamp‚1.7858141211585743e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8075951635114 trainer/global_step‚9199 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62583222 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 158286360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚734 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61126820 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 D>DeviceStatsMonitor.on_train_batch_end/sÄR>› egment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚157955929116672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚97159540 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚97159540 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36226734 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26356490 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚154030870193792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚166643091897856 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 158296032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8095852326400 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 158296032 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚158328506011648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚61126820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚221 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8076037438350 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62583445 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8314714456064 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8095764889600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚157955912077312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8314585886208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61126822 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚154030887233152%™Ç3á&·ÛM O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚166643245371392 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚158328530915328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚166051781443072 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8095764889600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚166051781443072 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8314585886208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61126822 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚154030887233152 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚97169210 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26356711 % _timestamp‚1.7858141211585743e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8075951635114 trainer/global_step‚9199 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62583222 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 158286360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚734 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61126820 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚157955929116672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚97159540 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚97159540 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36226734 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26356490 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚154030870193792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚166643091897856 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 158296032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8095852326400 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 158296032 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚158328506011648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚61126820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚221 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8076037438350 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62583445 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8314714456064 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8095764889600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚157955912077312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚166935964870144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8138968835072 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚158797100504064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚97677019 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 159129346 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚158797083464704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚167530697268224 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚166935964870144 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62916716 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 159139004 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.onæÅšô_train_batch_start/requested_bytes.large_pool.allocated‚154851155676832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162970204105356 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚9249 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚159171679638528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚159171654734848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚97677019 % _timestamp‚1.7858141413656542e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚166935964870144 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62916716 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 159139004 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚154851155676832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162970204105356 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚9249 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚159171679638528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚159171654734848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚97677019 % _timestamp‚1.7858141413656542e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚166935964870144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8138968835072 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚158797100504064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚97677019 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 159129346 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚158797083464704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚167530697268224 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8359918779904 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 159156184 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61458972 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚97687542 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 159156184 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26499965 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36423582 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚162987821751010 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62923547 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚167548645770240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8139848461824 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚159188861851648 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚167548805535232 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚158814267267072 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26500226 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8359783918592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚738 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61458970 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚166954011252736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8139761025024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚97687542 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚166954115728896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 159146512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚97697212 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8359918779904 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 159156184 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61458972 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚97687542 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 159156184 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26499965 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36423582 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚162987821751010 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62923547 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚167548645770240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚261 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8139848461824 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚159188861851648 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚167548805535232 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚158814267267072 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26500226 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8359783918592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚738 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61458970 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚166954011252736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8139761025024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚97687542 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚166954115728896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 159146512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚97697212 PÇÎÁ % _timestamp‚1.7858141622581336e+09 _runtime‚3907.448066677 _step‚740 & lr-AdamW/pg1‚0.0002396555621268614 & lr-AdamW/pg2‚0.0002396555621268614 trainer/global_step‚9299kÙ]¹ÌÏÆ % _timestamp‚1.7858141622581336e+09 _runtime‚3907.448066677 _step‚740 & lr-AdamW/pg1‚0.0002396555621268614 & lr-AdamW/pg2‚0.0002396555621268614 trainer/global_step‚9299ä$_²2Ð- _wandbruntime‚3907  _runtime‚3907Uê2vÒ'ÑÌO trainer/global_step‚9299 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26640696 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚163850998342380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8182877540864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚167838299155968 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚159655421615104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚159655438654464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36616497 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858141622590458e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚167838299155968 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8162851088908 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚61784477 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61784477 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚63257441 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚160032010574848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚155688147253472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8182964970496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36616495 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚160032035478528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61784479 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚168436231767040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8404196288512 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚159655438654464 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚167838403624960 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250T–‰-×'ÒÑO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26640696 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚163850998342380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8182877540864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚167838299155968 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚159655421615104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚159655438654464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36616497 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 % _timestamp‚1.7858141622590458e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚167838299155968 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8162851088908 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚61784477 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61784477 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚63257441 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚160032010574848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚155688147253472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8182964970496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36616495 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚160032035478528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61784479 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚168436231767040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8404196288512 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚159655438654464 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚167838403624960 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 trainer/global_step‚9299åP(ˆø2Ó- _wandbruntime‚3907  _runtime‚3907-è¡!ÔÖM I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚159672588378112 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8183844597248 _runtime‚3907.805122501 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8163728562130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 160006664 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚98225214 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8405098026496 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 _step‚742 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚159672588378112 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚160049217691648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 160006664 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚163868615987922 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63263925 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8183844597248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61791122 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7858141626148405e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 160016336 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8183757160448 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36620432 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61791122 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚98215544 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚63264175 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚167856450014720 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 160016336 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚61791120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26643741 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚98215544 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚167856450014720 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚155704904465152 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚98225214 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/request#­¯ ;ed_bytes.small_pool.allocated‚ 8163814365366 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚159672605417472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚167856345538560 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚159672605417472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26643493 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8183757160448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596->á&ÕÛM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61791122 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚98215544 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚63264175 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚167856450014720 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 160016336 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚61791120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26643741 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚98215544 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚167856450014720 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚155704904465152 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚98225214 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8163814365366 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚159672605417472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚167856345538560 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚159672605417472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26643493 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8183757160448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚159672588378112 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8183844597248 _runtime‚3907.805122501 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8163728562130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 160006664 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚98225214 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8405098026496 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 _step‚742 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚159672588378112 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚160049217691648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 160006664 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚163868615987922 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63263925 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8183844597248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61791122 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7858141626148405e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 160016336 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8183757160448 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36620432 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248æ̱#'2Ö- _wandbruntime‚3907  _runtime‚3907¯Þ± ׃ /loss/FRt1/detection_loss‚2.2412487794174236 /loss/FBt1/detection_loss‚2.3770226108926265 epoch‚0 .loss/FRt1/assignment_loss‚2.43475280718857 /loss/FRt2/assignment_loss‚2.794256450401412 2loss/SRqqt2/assignment_loss‚0.7202402752062137 2loss/FBt1/assignment_loss‚0.016272626022356462 4loss/FBt2/assignment_loss‚2.2360510552843495e-05 1loss/SRqqt2/detection_loss‚1.7873605670576747 trainer/global_step‚9299 2loss/SRqqt1/assignment_loss‚1.0546824919007771 /loss/FRt2/detection_loss‚2.9499523014658027 1loss/SRqqt1/detection_loss‚0.9992747956795709 % _timestamp‚1.7858141626160526e+09 _runtime‚3907.805832126 /loss/FBt2/detection_loss‚1.0325353356078266 &loss/total_loss‚18.407621401350806 _step‚743 Ãõ"؈ 1loss/SRqqt1/detection_loss‚0.9992747956795709 % _timestamp‚1.7858141626160526e+09 _runtime‚3907.805832126 /loss/FBt2/detection_loss‚1.0325353356078266 &loss/total_loss‚18.407621401350806 _step‚743 /loss/FRt1/detection_loss‚2.2412487794174236 /loss/FBt1/detection_loss‚2.3770226108926265 epoch‚0 .loss/FRt1/assignment_loss‚2.43475280718857 /loss/FRt2/assignment_loss‚2.794256450401412 2loss/SRqqt2/assignment_loss‚0.7202402752062137 2loss/FBt1/assignment_loss‚0.016272626022356462 4loss/FBt2/assignment_loss‚2.2360510552843495e-05 1loss/SRqqt2/detection_loss‚1.7873605670576747 trainer/global_step‚9299 2loss/SRqqt1/assignment_loss‚1.0546824919007771 /loss/FRt2/detection_loss‚2.9499523014658027ç±Ê(]Ù‚(:Ò  ™¹ÅÓë鹃 disk.drbd1817.in‚0 disk.drbd1356.out‚0$ memory_percent‚41.24503135681152 disk.drbd1817.out‚0% disk.nvme0n1.out‚3935.59033203125 disk.drbd1356.in‚0 disk.drbd1828.in‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.nvme0n1.in‚ 431.34375 disk.drbd2041.out‚0 disk.drbd2118.in‚0 disk.drbd1285.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1552.in‚0 proc.cpu.threads‚29 disk.drbd1673.out‚0 disk.drbd1938.out‚0 disk.drbd1938.in‚0 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.drbd1673.in‚0 disk.drbd1226.out‚0 disk.drbd2078.out‚0 disk.drbd2111.out‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda2.out‚165.625 disk.rbd1.out‚ 14.0224609375 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd2118.out‚0 network.sent‚ 8.663071e+06 disk.drbd1931.in‚0) disk./.usagePercent‚32.95958918777881 disk.sda2.in‚11.625 disk.rbd1.in‚0 disk.drbd1584.out‚0' proc.memory.availableMB‚ 38505.65625* proc.memory.percent‚17.172330617904663 network.recv‚578009 disk.sda.out‚165.625 disk.dm-0.in‚ 431.34375" disk.dm-0.out‚3935.59033203125 cpu‚13.745416283588039$ disk./.usageGB‚589.3509292602539 disk.drbd1584.in‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.sda1.out‚0 disk.drbd1289.out‚0 disk.sda.in‚11.625 disk.drbd1965.in‚0 disk.rbd0.out‚0 disk.drbd1510.in‚0 £—'œÚ‚(:‘ ™¹ÅÓÓ滃 gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.529' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.50966666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0K½c2Û- _wandbruntime‚3928  _runtime‚3928ù’®4ÇÜÁ & lr-AdamW/pg1‚0.0002409441714511267 & lr-AdamW/pg2‚0.0002409441714511267 trainer/global_step‚9349 % _timestamp‚1.7858141829560568e+09 _runtime‚3928.145997153 _step‚744ݳÿÌÝÆ trainer/global_step‚9349 % _timestamp‚1.7858141829560568e+09 _runtime‚3928.145997153 _step‚744 & lr-AdamW/pg1‚0.0002409441714511267 & lr-AdamW/pg2‚0.0002409441714511267èÒ¼2Þ- _wandbruntime‚3928  _runtime‚3928Ž‹CÒ'ßÌO F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚156525172908832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚98733023 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36813347 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚98742679 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚63595496 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚169341766764032 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚160892391318528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚164731998264372 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚168740737910784 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8226873676288 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8226961105920 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26782151 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _runtime‚3928.150642053 % _timestamp‚1.7858141829570045e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 160849650 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62116627 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚168740633441792 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚160513776804864 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚9349 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚168740633441792 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚160513759765504 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚168740737910784 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62116627 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032ðúSg~àÑO G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚98733023 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36813347 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚98742679 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚63595496 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚169341766764032 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚160892391318528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚164731998264372 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚168740737910784 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8226873676288 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8226961105920 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26782151 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 _runtime‚3928.150642053 % _timestamp‚1.7858141829570045e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 160849650 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62116627 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚168740633441792 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚243 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚160513776804864 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 LDDeviceStatsMonitor.on_traiÃÉúÚY n_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚9349 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚168740633441792 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚160513759765504 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚168740737910784 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62116627 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚156525172908832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0é¾*u2á- _wandbruntime‚3928  _runtime‚3928ãQoQÜ&âÖM TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚98753216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚164749615910026 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858141833084772e+09 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚169359716517376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚168758784300544 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚3928.498762831 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚160530943567872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚63602464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚98743546 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 160876488 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8450279944192 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚9349 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8227840732672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36817282 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8207702828874 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8227753295872 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 160866816 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26784926 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26785180 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚160530926528512 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚156541913081152 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62123270 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8207617025638 _step‚746 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚160530926528512 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚169359878379520 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36817284 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚160530943567872 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚168758784300544 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8450142985728 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚168758679824384 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8227753295872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62123270 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚160909598435328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚156541896041792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 160866816 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63602208 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62123272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚160909573531648 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8450142985728 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚168758679824384 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8227753295872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62123270 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚160909598435328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚156541896041792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 160866816 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63602208 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62123272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚160909573531648 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858141833084772e+09 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚169359716517376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚168758784300544 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚3928.498762831 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚160530943567872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚63602464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚98743546 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 160876488 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8450279944192 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚9349 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8227840732672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36817282 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8207702828874 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8227753295872 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 160866816 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26784926 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26785180 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚160530926528512 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚156541913081152 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62123270 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8207617025638 _step‚746 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚160530926528512 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚169359878379520 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36817284 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚160530943567872 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚168758784300544 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚169642967727616 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26924055 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62448779 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚161372097915904 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚99270681 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚157362164485472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8250713819048 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚165612792501396 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 161709802 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62448777 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚161752722254848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.o‡ÿÏn_train_batch_start/allocated_bytes.small_pool.allocated‚ 8270957241344 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170247166111232 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8270869811712 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚161372114955264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3948.891758944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚169643072196608 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚749 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8270957241344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚99261025 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚169643072196608 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚169642967727616 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 161719460 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚165612895343880 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚99261025 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37010195 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26924312 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚9399 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392®N§Ê×'ðÑO VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8270869811712 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8250628015924 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 % _timestamp‚1.7858142037015696e+09 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚63934250 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37010197 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8270957241344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚99261025 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚169643072196608 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚169642967727616 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 161719460 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚165612895343880 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚99261025 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37010195 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26924312 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚9399 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚170247323786240 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8494576627712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚157362181524832 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚169642967727616 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26924055 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62448779 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚161372097915904 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚99270681 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚157362164485472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8250713819048 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚165612792501396 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 161709802 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62448777 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚161752722254848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8270957241344 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170247166111232 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8270869811712 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚161372114955264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3948.891758944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚169643072196608 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚259 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚749 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 trainer/global_step‚9399 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26926892 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚165630512989534 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚170265433075200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8251591292382 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚169661118586368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8271749431296 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚99271548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858142040587878e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62455422 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚161389281718272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚165630410146938 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 161726968 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚750 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8495478799872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8271749431296 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚161769954275328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 161726968 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚99271548 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62455422 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63941024 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚169661014110208 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 161736640 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8251505489146 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26927136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8271836868096 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0Uº.)óÛM M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8271836868096 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 trainer/global_step‚9399 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26926892 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚165630512989534 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚244 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚170265433075200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8251591292382 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚169661118586368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8271749431296 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚99271548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 % _timestamp‚1.7858142040587878e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62455422 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚161389281718272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚165630410146938 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 161726968 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚750 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8495478799872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8271749431296 LDDeviceStatsMonitor.on_train_batch_end/inac¿v_¸tive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚161769954275328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 161726968 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚99271548 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62455422 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63941024 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚169661014110208 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 161736640 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8251505489146 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26927136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121îÏð9€2ô- _wandbruntime‚3949  _runtime‚3949 õˆÙõ‚ epoch‚0 /loss/FRt1/detection_loss‚2.3274135975896018 .loss/FBt2/detection_loss‚1.343736650591547 % _timestamp‚1.7858142040599885e+09 /loss/FRt1/assignment_loss‚2.345947107543116 1loss/SRqqt1/assignment_loss‚1.006464068029373 3loss/FBt1/assignment_loss‚0.0024956647584336417 /loss/FRt2/detection_loss‚2.4176467496861695 0loss/SRqqt1/detection_loss‚1.086964158741243 &loss/total_loss‚17.081646555013588 _step‚751 2loss/SRqqt2/assignment_loss‚1.0212874679572204 4loss/FBt2/assignment_loss‚0.00021246867570471036 1loss/SRqqt2/detection_loss‚1.5677555073391307 /loss/FBt1/detection_loss‚1.1832922163995958 trainer/global_step‚9399 _runtime‚3949.249764971 /loss/FRt2/assignment_loss‚2.778430897702453{¬=¼ ö‡ epoch‚0 /loss/FRt1/detection_loss‚2.3274135975896018 .loss/FBt2/detection_loss‚1.343736650591547 % _timestamp‚1.7858142040599885e+09 /loss/FRt1/assignment_loss‚2.345947107543116 1loss/SRqqt1/assignment_loss‚1.006464068029373 3loss/FBt1/assignment_loss‚0.0024956647584336417 /loss/FRt2/detection_loss‚2.4176467496861695 0loss/SRqqt1/detection_loss‚1.086964158741243 &loss/total_loss‚17.081646555013588 _step‚751 2loss/SRqqt2/assignment_loss‚1.0212874679572204 4loss/FBt2/assignment_loss‚0.00021246867570471036 1loss/SRqqt2/detection_loss‚1.5677555073391307 /loss/FBt1/detection_loss‚1.1832922163995958 trainer/global_step‚9399 _runtime‚3949.249764971 /loss/FRt2/assignment_loss‚2.778430897702453ï4ïã÷‚(:… ƹÅÓÝ«ªƒ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.893' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.631 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ZT™oø‚(:ä  ƹÅÓÁ™Êƒ disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1356.out‚0 disk.drbd2041.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 network.sent‚ 8.756299e+06 disk.drbd1226.in‚0 disk.sda1.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1965.out‚0 disk.drbd1817.out‚0* proc.memory.percent‚17.172330617904663 disk.sda2.out‚ 169.14453125) disk./.usagePercent‚32.95958897444097$ disk./.usageGB‚589.3509254455566 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd1510.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.sda2.in‚ 11.6328125$ memory_percent‚41.24569892883301 cpu‚13.744627733392889' proc.memory.availableMB‚ 38505.21875 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚ 431.34375 proc.cpu.threads‚29 disk.drbd1584.in‚0" disk.dm-0.out‚3939.41748046875 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.dm-0.in‚ 431.34375 disk.drbd2118.out‚0 disk.drbd1938.out‚0 network.recv‚582066 disk.drbd1931.in‚0 disk.drbd1285.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.drbd1817.in‚0 disk.drbd1938.in‚0 disk.rbd1.out‚ 14.1708984375 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd1673.in‚0 disk.drbd2078.in‚0 disk.drbd2111.in‚0 disk.sda.out‚ 169.14453125 disk.drbd2078.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0% disk.nvme0n1.out‚3939.41748046875 disk.drbd1510.in‚0 disk.drbd2118.in‚0²g‡(2ù- _wandbruntime‚3969  _runtime‚3969Ë/­EÈú trainer/global_step‚9449 $ _timestamp‚1.785814224653795e+09 _runtime‚3969.845622249 _step‚752 ' lr-AdamW/pg1‚0.00024352139009965732 ' lr-AdamW/pg2‚0.00024352139009965732~lü ÍûÇ trainer/global_step‚9449 $ _timestamp‚1.785814224653795e+09 _runtime‚3969.845622249 _step‚752 ' lr-AdamW/pg1‚0.00024352139009965732 ' lr-AdamW/pg2‚0.00024352139009965732ð.þ,÷2ü- _wandbruntime‚3969  _runtime‚3969:¤[Ò'ýÌO TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚162230453105664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚99798683 _runtime‚3969.845700041 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚162230453105664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8314865947136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62780927 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚158199190140832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚166493792423388 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858142246544747e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62780929 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚171152828154368 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8294516479432 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚99789027 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚170545406482432 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162613078094848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚158199173101472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8314865947136 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚170545302013440 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 162579612 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚166493689580904 _step‚753 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚99789027 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚171152670479360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚170545302013440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8314953376768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8294602282556 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62780927 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚99798683 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚162230436066304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27067766 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 162569954 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62780929 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚64274813 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032x7{×'þÑO ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8294602282556 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62780927 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚99798683 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚162230436066304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27067766 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 162569954 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62780929 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚64274813 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚162230453105664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚99798683 _runtime‚3969.845700041 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚162230453105664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8314865947136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62780927 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚158199190140832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚166493792423388 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858142246544747e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62780929 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚171152828154368 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8294516479432 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚99789027 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚170545406482432 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162613078094848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚158199173101472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8314865947136 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚170545302013440 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 162579612 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚166493689580904 _step‚753 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚99789027 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚171152670479360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚170545302013440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8314953376768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚257ñÕnº2ÿ- _wandbruntime‚3970  _runtime‚39703+?Ì€ÕM L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 162596792 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _step‚754 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8295393952654 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚162247602829312 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64281322 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚99809220 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8315833003520 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 162587120 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚99809220 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62787570 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27070598 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚162247619868672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8315833003520 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚162247602829312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8315745566720 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚170563452872192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.allì J.allocated‚64281582 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62787572 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚166511307226446 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚170563348396032 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 162596792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚99799550 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 162587120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37210984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚171170935337984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8540625222656 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8540492458496 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37210982 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚158215930313152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚3970.195089438 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62787570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚162630285211648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 $ _timestamp‚1.785814225004818e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8315745566720 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚260 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚158215913273792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚99799550 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0="óà&ÚM TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64281322 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚99809220 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8315833003520 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 162587120 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚99809220 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62787570 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27070598 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚162247619868672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8315833003520 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚162247602829312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8315745566720 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚170563452872192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚64281582 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62787572 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚166511307226446 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚170563348396032 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 162596792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚99799550 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 162587120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37210984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚171170935337984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8540625222656 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8540492458496 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37210982 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚158215930313152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚3970.195089438 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62787570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚162630285211648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 $ _timestamp‚1.785814225004818e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8315745566720 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚260 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚158215913273792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚99799550 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 162596792 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _step‚754 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8295393952654 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚162247602829312 EÖ'ÐO ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚163088791256064 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚171447636299264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8584916129280 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8358862082560 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27209138 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚163088791256064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚64613033 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37403897 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚159036198756832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 100326685 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚172058374967808 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚167374586660412 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚171447740768256 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63113079 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8358949512192 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚163473433934848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63113079 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚163088774216704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚159036181717472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 163439764 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3991.089631995 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8358949512192 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100317029 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8584777066496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 163430106 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚172058211001344 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚171447636299264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚64613304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291y®ß©ŽÕO UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130øor<2' LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3991.089631995 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8358949512192 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100317029 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8584777066496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 163430106 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚172058211001344 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚171447636299264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚64613304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚163088791256064 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚171447636299264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8584916129280 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8358862082560 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27209138 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚163088791256064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚64613033 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37403897 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚159036198756832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 100326685 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚172058374967808 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚167374586660412 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚171447740768256 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63113079 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8358949512192 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚163473433934848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63113079 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚163088774216704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚159036181717472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 163439764 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _step‚758 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8585681965568 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64619830 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27212241 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 163456944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 163447272 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63119722 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63119720 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 100337222 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 163456944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚163490641051648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚159052938929152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37407834 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8339282416162 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚159052921889792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚171465787158016 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27211998 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8339368219398 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8585814729728 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 163456944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 163447272 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63119722 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63119720 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 100337222 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 163456944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚163490641051648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚159052938929152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37407834 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8339282416162 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚159052921889792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚171465787158016 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27211998 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8339368219398 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8585814729728 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _step‚758 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8585681965568 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64619830 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27212241 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚172349970585088 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 _runtime‚4011.471669621 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚172963900248064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚64954231 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚172349970585088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚172963740475904 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63445227 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8629950701056 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 100845031 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚168255483739920 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8402858217984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚172350075054080 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚164333814678528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8402858217984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8630085569536 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100845031 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚163947129406464 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8402945647616 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8382293406448 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚64954231 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚172349970585088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚172963740475904 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63445227 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8629950701056 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 100845031 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚168255483739920 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8402858217984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚172350075054080 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚164333814678528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8402858217984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8630085569536 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100845031 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚163947129406464 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8402945647616 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8382293406448 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚172349970585088 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 _runtime‚4011.471669621 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚172963900248064ùCeä2- _wandbruntime‚4011  _runtime‚4011O¢hß&žÙM LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64960749 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚163964296169472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 100865224 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63451872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚172982007636480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚172368121443840 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚163964279130112 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚168273101385462 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 164317096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8383256682906 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚163964279130112 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚164350996891648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8630985841152 _step‚762 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚164351021795328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8403825274368 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚237 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27356067 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _runtime‚ 4011.81803775 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 100855554 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚172368016967680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚9549 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63451872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8403737837568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37604682 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 100865224 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27356302 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63451870 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 100855554 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858142666277623e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8403737837568ú&ðËJŸÞM =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚163964279130112 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚164350996891648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8630985841152 _step‚762 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚164351021795328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 2537553ú„c€š#92 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8403825274368 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚237 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27356067 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 _runtime‚ 4011.81803775 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 100855554 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚172368016967680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚9549 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63451872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8403737837568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37604682 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 100865224 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27356302 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63451870 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 100855554 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858142666277623e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8403737837568 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64960749 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚163964296169472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 100865224 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63451872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚172982007636480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚172368121443840 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚163964279130112 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚168273101385462 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 164317096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8383256682906úórè2 - _wandbruntime‚4011  _runtime‚4011:{s¡ 0loss/FRt1/assignment_loss‚2.1633088522487216 .loss/FRt2/detection_loss‚2.390425712366899 /loss/FBt2/detection_loss‚1.0616623517125845 trainer/global_step‚9549 3loss/FBt2/assignment_loss‚0.0004385145180293056 -loss/FRt1/detection_loss‚2.45601849331532 1loss/SRqqt2/detection_loss‚1.5123851816406695 % _timestamp‚1.7858142666289656e+09 /loss/FRt2/assignment_loss‚2.777791180875566 2loss/FBt1/assignment_loss‚0.022597275768540013 1loss/SRqqt1/detection_loss‚0.9860916566239656 /loss/FBt1/detection_loss‚0.8804689559287259 2loss/SRqqt1/assignment_loss‚0.8587318993391074 2loss/SRqqt2/assignment_loss‚0.7493010511053568 &loss/total_loss‚15.859221125443485 epoch‚0 _runtime‚4011.818760349 _step‚763IÉ»4 ¢† 2loss/FBt1/assignment_loss‚0.022597275768540013 1loss/SRqqt1/detection_loss‚0.9860916566239656 /loss/FBt1/detection_loss‚0.8804689559287259 2loss/SRqqt1/assignment_loss‚0.8587318993391074 2loss/SRqqt2/assignment_loss‚0.7493010511053568 &loss/total_loss‚15.859221125443485 epoch‚0 _runtime‚4011.818760349 _step‚763 0loss/FRt1/assignment_loss‚2.1633088522487216 .loss/FRt2/detection_loss‚2.390425712366899 /loss/FBt2/detection_loss‚1.0616623517125845 trainer/global_step‚9549 3loss/FBt2/assignment_loss‚0.0004385145180293056 -loss/FRt1/detection_loss‚2.45601849331532 1loss/SRqqt2/detection_loss‚1.5123851816406695 % _timestamp‚1.7858142666289656e+09 /loss/FRt2/assignment_loss‚2.777791180875566ûžµÝ›£‚(: ‚ºÅÓâ爃 gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚95.778' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚31.926000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0±@¢ìo¤‚(:ä  ‚ºÅÓ÷ïσ disk.drbd1552.in‚0 disk.drbd1550.in‚0) proc.memory.availableMB‚ 38505.5234375 disk.drbd1673.in‚0 disk.dm-0.in‚ 440.05859375 disk.drbd1285.in‚0 disk.drbd1828.out‚0 network.sent‚ 8.894341e+06 disk.drbd1817.in‚0 disk.drbd1673.out‚0 disk.drbd1550.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.sda.in‚ 11.6328125 cpu‚13.743053187677754# disk./.usageGB‚589.350944519043 disk.drbd1584.out‚0 disk.drbd1965.out‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.sda2.in‚ 11.6328125 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.drbd1828.in‚0 disk.dm-0.out‚ 3941.8515625$ memory_percent‚41.24523401260376 disk.sda.out‚ 171.94921875 disk.drbd2078.in‚0* proc.memory.percent‚17.172330617904663 network.recv‚587567 disk.drbd2118.in‚0 disk.rbd1.out‚14.26611328125 disk.drbd1289.out‚0 proc.cpu.threads‚29) disk./.usagePercent‚32.95959004113017 disk.sda1.out‚0 disk.drbd1938.out‚0 disk.nvme0n1.in‚ 440.05859375 disk.sda2.out‚ 171.94921875 disk.drbd1356.in‚0 disk.drbd2078.out‚0 disk.drbd1289.in‚0 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.drbd1226.in‚0 disk.drbd1510.in‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.drbd1931.in‚0 disk.drbd2111.in‚0 disk.drbd1938.in‚0! disk.nvme0n1.out‚ 3941.8515625 disk.drbd1584.in‚0 disk.drbd1226.out‚0 disk.rbd0.out‚0 disk.drbd2111.out‚0‹×2¥- _wandbruntime‚4032  _runtime‚4032ã€%ÖǦÁ & lr-AdamW/pg1‚0.0002473872180724532 & lr-AdamW/pg2‚0.0002473872180724532 trainer/global_step‚9599 % _timestamp‚1.7858142871923046e+09 _runtime‚4032.382253307 _step‚764 LÁÌ§Æ & lr-AdamW/pg1‚0.0002473872180724532 & lr-AdamW/pg2‚0.0002473872180724532 trainer/global_step‚9599 % _timestamp‚1.7858142871923046e+09 _runtime‚4032.382253307 _step‚764ü–áÇ©2¨- _wandbruntime‚4032  _runtime‚4032Ü*ýÖ'©ÐO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63777379 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚160710215988832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63777377 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 101382689 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 101373033 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858142871932306e+09 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚65294139 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚165194145614848 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37797595 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚164805450517504 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚173869249649152 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63777377 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 165160068 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 101382689 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8446941783040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚160710198949472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚4032.386636146 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚173252304870912 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚164805467556864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _step‚765 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 165150410 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚164805467556864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37797597 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚247 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 101373033 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8675104034304 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121häö#ªÕO YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚164805450517504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚173252304870912 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8675240999936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8426267673080 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8426181869956 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63777379 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚65293890 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚173869411518464 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚165194170518528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚249 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8446854353408 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27496295 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚169136483661912 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚169136380819428 trainer/global_step‚9599 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚173252409339904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8446941783040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27496542 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚173252409339904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7858142871932306e+09 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚65294139 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚165194145614848 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37797595 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚164805450517504 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚173869249649152 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63777377 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 165160068 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 101382689 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8446941783040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚160710198949472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚4032.386636146 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚173252304870912 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚164805467556864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _step‚765 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 165150410 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚164805467556864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37797597 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚247 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 101373033 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8675104034304 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63777379 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚160710215988832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63777377 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 101382689 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 101373033 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G>´‡<;åDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚164822617280512 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚173270455729664 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 101393226 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8676008859648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8676141623808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 165167576 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚9599 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 165177248 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚169154101307566 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 165177248 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚165211377635328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65300643 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 _step‚766 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65300886 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27499111 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚169153998464970 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚164822634319872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8447821409792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚165211352731648 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858142875431025e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚173887519259136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚160726956161152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63784022 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚164822617280512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚173887361591296 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 165177248 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚165211377635328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65300643 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 _step‚766 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65300886 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27499111 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚169153998464970 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚164822634319872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8447821409792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚165211352731648 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 % _timestamp‚1.7858142875431025e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚173887519259136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚160726956161152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚241 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63784022 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚164822617280512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚173887361591296 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚164822617280512 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚173270455729664 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 101393226 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8676008859648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8676141623808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 165167576 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚9599 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 165177248 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚169154101307566þ~õ<2®- _wandbruntime‚4032  _runtime‚4032òk  ¯„ /loss/FBt2/detection_loss‚1.3504417710514232 _step‚767 0loss/FRt1/assignment_loss‚2.1619447335390976 2loss/SRqqt1/assignment_loss‚0.9852885958324704 1loss/SRqqt2/assignment_loss‚0.931759194344661 3loss/FBt2/assignment_loss‚4.844159758089393e-06 1loss/SRqqt1/detection_loss‚1.0809338952449186 /loss/FBt1/detection_loss‚1.0874996019903427 trainer/global_step‚9599 0loss/FRt2/assignment_loss‚2.6475358247756957 /loss/FRt2/detection_loss‚2.6841138978799184 1loss/SRqqt2/detection_loss‚1.2912295653910961 %loss/total_loss‚16.93961896516077 epoch‚0 4loss/FBt1/assignment_loss‚2.7149381671766658e-05 .loss/FRt1/detection_loss‚2.718839891569715 % _timestamp‚1.7858142875443096e+09 _runtime‚4032.734139346ÂÊÝ°‰ 0loss/FRt2/assignment_loss‚2.6475358247756957 /loss/FRt2/detection_loss‚2.6841138978799184 1loss/SRqqt2/detection_loss‚1.2912295653910961 %loss/total_loss‚16.93961896516077 epoch‚0 4loss/FBt1/assignment_loss‚2.7149381671766658e-05 .loss/FRt1/detection_loss‚2.718839891569715 % _timestamp‚1.7858142875443096e+09 _runtime‚4032.734139346 /loss/FBt2/detection_loss‚1.3504417710514232 _step‚767 0loss/FRt1/assignment_loss‚2.1619447335390976 2loss/SRqqt1/assignment_loss‚0.9852885958324704 1loss/SRqqt2/assignment_loss‚0.931759194344661 3loss/FBt2/assignment_loss‚4.844159758089393e-06 1loss/SRqqt1/detection_loss‚1.0809338952449186 /loss/FBt1/detection_loss‚1.0874996019903427 trainer/global_step‚9599ÿ"póTu±‚(:ê  ‘ºÅÓðÐÙƒ disk.drbd1356.in‚0 disk.drbd1289.in‚0 disk.nvme0n1.in‚ 440.05859375 proc.cpu.threads‚29 network.recv‚589237 disk.drbd1356.out‚0 disk.drbd2078.in‚0 disk.rbd0.in‚0! disk.dm-0.out‚3942.6748046875 disk.drbd2111.out‚0 disk.drbd2118.out‚0 cpu‚13.742742558489999 network.sent‚ 8.939346e+06 disk.drbd1965.in‚0 disk.drbd1510.in‚0 disk.drbd1938.out‚0* proc.memory.percent‚17.172330617904663 disk.drbd2041.in‚0 disk.drbd1285.in‚0 disk.sda1.in‚0 disk.sda2.in‚ 11.6328125 disk.sda.in‚ 11.6328125 disk.sda.out‚ 172.32421875 disk.drbd1673.out‚0 disk.drbd2041.out‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.sda1.out‚0$ memory_percent‚41.24530553817749 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd1550.out‚0 disk.drbd1828.in‚0 disk.drbd2118.in‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 172.32421875 disk.dm-0.in‚ 440.05859375 disk.rbd1.out‚ 14.3291015625 disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd2111.in‚0$ disk.nvme0n1.out‚3942.6748046875$ disk./.usageGB‚589.3509254455566 disk.drbd1584.out‚0 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0 disk.rbd1.in‚0) disk./.usagePercent‚32.95958897444097 disk.drbd1931.out‚0) proc.memory.availableMB‚ 38505.4765625$ proc.memory.rssMB‚11254.05859375 disk.drbd1584.in‚0 disk.drbd1965.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0Š¸Ðœ²‚(:‘ ‘ºÅÓ¦æ߃ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚126.844' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.28133333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0õéÆ‚œ³‚(:‘  ºÅÓ¦©©ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚120.598' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.19933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ó¶9­u´‚(:ê   ºÅÓÍÝçƒ disk.drbd1938.in‚0$ memory_percent‚41.24607443809509 disk.drbd1931.out‚0 disk.sda.out‚ 172.91796875 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚ 440.05859375 disk.drbd2118.out‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1828.in‚0 disk.drbd2041.out‚0 disk.drbd1226.out‚0 disk.drbd1673.out‚0 disk.drbd1584.in‚0 disk.drbd2078.out‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.sda2.out‚ 172.91796875 network.sent‚ 8.984301e+06) disk./.usagePercent‚32.95959004113017 disk.drbd1965.in‚0 disk.drbd1552.out‚0 disk.sda2.in‚ 11.6328125* proc.memory.percent‚17.172330617904663 cpu‚13.742616294269036# disk./.usageGB‚589.350944519043 disk.drbd1285.in‚0 disk.drbd1285.out‚0 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd1965.out‚0 disk.drbd1673.in‚0 disk.drbd1356.out‚0 disk.sda.in‚ 11.6328125 disk.drbd2078.in‚0 disk.drbd1817.out‚0 disk.drbd1828.out‚0 disk.drbd1584.out‚0$ proc.memory.rssMB‚11254.05859375 disk.drbd2041.in‚0! disk.dm-0.out‚3942.8623046875 disk.drbd1938.out‚0 disk.drbd1510.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 network.recv‚590701 disk.drbd1226.in‚0 disk.drbd1550.out‚0 disk.drbd1510.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 14.3291015625 disk.sda1.in‚0 disk.drbd1552.in‚0$ disk.nvme0n1.out‚3942.8623046875* proc.memory.availableMB‚38504.97265625 proc.cpu.threads‚29 disk.drbd1356.in‚0 disk.drbd1931.in‚0 disk.drbd2111.in‚0ù`3ý2µ- _wandbruntime‚4053  _runtime‚4053ÌXYɶà ' lr-AdamW/pg1‚0.00024867582739671854 ' lr-AdamW/pg2‚0.00024867582739671854 trainer/global_step‚9649 % _timestamp‚1.7858143081279852e+09 _runtime‚4053.318184198 _step‚7682o Î·È _step‚768 ' lr-AdamW/pg1‚0.00024867582739671854 ' lr-AdamW/pg2‚0.00024867582739671854 trainer/global_step‚9649 % _timestamp‚1.7858143081279852e+09 _runtime‚4053.318184198€{”ƒ2¸- _wandbruntime‚4053  _runtime‚4053ƒÔŠæL¹ÐO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27639978 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚174154639156736 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64109529 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8720446738944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 166010562 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGÈí˜>ŠDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27639723 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚161547207565472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8470156136588 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37994447 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37994445 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚174154639156736 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8490850488832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 101901035 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 166020220 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚166054501454848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170017277898936 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170017380741420 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 101901035 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27639723 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚161547207565472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8470156136588 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37994447 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37994445 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚174154639156736 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8490850488832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 101901035 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 166020220 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚166054501454848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170017277898936 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170017380741420 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 101901035 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8720446738944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 166010562 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚174792932640256 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚166071708571648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚165680955430912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8491817545216 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚174172790015488 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27642821 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27642583 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚174793079822336 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 101911558 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64116172 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8491730108416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8471033609922 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8470947806686 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858143084794695e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65640965 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 166037400 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚165680955430912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 166027728 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 122278400 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 166037400 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚1656809724702728äÛçg½ÞM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚174792932640256 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚166071708571648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚165680955430912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8491817545216 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚174172790015488 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27642821 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27642583 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚174793079822336 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 101911558 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64116172 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8491730108416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8471033609922 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8470947806686 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 % _timestamp‚1.7858143084794695e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65640965 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 166037400 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚165680955430912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 166027728 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 122278400 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 166037400 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚165680972470272 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 101921228 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37998384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚170034895544478 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚161563964777152 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚770 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚165680972470272 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚238 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.lar†á‹œ}ge_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64116170 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8491817545216 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65641205 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 101911558 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8721224068608 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8491730108416 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚ 4053.66976347 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0‚“zM2¾- _wandbruntime‚4053  _runtime‚4053ÎQcd ¿… 2loss/SRqqt1/assignment_loss‚0.9573350993632157 4loss/FBt2/assignment_loss‚1.1085325420197023e-05 1loss/SRqqt2/detection_loss‚1.3845133188722738 /loss/FRt1/assignment_loss‚2.391940054205275 0loss/FRt2/assignment_loss‚2.7823851537413713 3loss/FBt1/assignment_loss‚8.804036203408876e-06 epoch‚0 $ _timestamp‚1.785814308480697e+09 _runtime‚4053.670455612 2loss/SRqqt2/assignment_loss‚0.8228303414818122 /loss/FRt1/detection_loss‚2.1986761103124963 1loss/SRqqt1/detection_loss‚0.9901810497190658 &loss/total_loss‚15.828743103733323 /loss/FRt2/detection_loss‚2.5503847984279076 /loss/FBt1/detection_loss‚0.6642654520592519 /loss/FBt2/detection_loss‚1.0862118361890316 trainer/global_step‚9649 _step‚771Éþ–dÀŠ /loss/FRt2/detection_loss‚2.5503847984279076 /loss/FBt1/detection_loss‚0.6642654520592519 /loss/FBt2/detection_loss‚1.0862118361890316 trainer/global_step‚9649 _step‚771 2loss/SRqqt1/assignment_loss‚0.9573350993632157 4loss/FBt2/assignment_loss‚1.1085325420197023e-05 1loss/SRqqt2/detection_loss‚1.3845133188722738 /loss/FRt1/assignment_loss‚2.391940054205275 0loss/FRt2/assignment_loss‚2.7823851537413713 3loss/FBt1/assignment_loss‚8.804036203408876e-06 epoch‚0 $ _timestamp‚1.785814308480697e+09 _runtime‚4053.670455612 2loss/SRqqt2/assignment_loss‚0.8228303414818122 /loss/FRt1/detection_loss‚2.1986761103124963 1loss/SRqqt1/detection_loss‚0.9901810497190658 &loss/total_loss‚15.828743103733323ƒà‹(¥Á‚(:’ ¯ºÅÓ¸‘ü‚ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚123.901' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚41.300333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Öág‹n‚(:ã  ¯ºÅÓŸëă disk.drbd2078.in‚0 disk.drbd1828.in‚0 disk.drbd2111.in‚0 disk.drbd1552.out‚0 disk.drbd2078.out‚0 disk.drbd1931.out‚0 disk.drbd1285.in‚0 disk.drbd1356.in‚0 disk.drbd1931.in‚0$ proc.memory.rssMB‚11253.87109375 proc.cpu.threads‚29 disk.drbd1965.out‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 cpu‚13.742160536246436 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.drbd1938.out‚0 disk.sda.out‚ 173.2578125 disk.rbd0.in‚0$ disk./.usageGB‚589.3509254455566 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd1828.out‚0 disk.drbd1289.in‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1673.out‚0# memory_percent‚41.2451446056366 disk.rbd0.out‚0 disk.drbd1510.in‚0* proc.memory.availableMB‚38505.58203125) proc.memory.percent‚17.17204451560974 network.sent‚ 8.987462e+06 disk.dm-0.out‚ 3943.15234375 disk.sda2.out‚ 173.2578125 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.drbd1550.out‚0 disk.drbd1289.out‚0 network.recv‚591678 disk.drbd1965.in‚0 disk.sda1.in‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.sda.in‚ 11.6328125) disk./.usagePercent‚32.95958897444097 disk.sda2.in‚ 11.6328125 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd1938.in‚0 disk.drbd1226.in‚0 disk.drbd1285.out‚0 disk.dm-0.in‚ 440.05859375 disk.rbd1.out‚ 14.4775390625" disk.nvme0n1.out‚ 3943.15234375 disk.drbd2041.in‚0/Ǽ2Ã- _wandbruntime‚4074  _runtime‚4074Ña€uÉÄà ' lr-AdamW/pg1‚0.00024996443672098385 ' lr-AdamW/pg2‚0.00024996443672098385 trainer/global_step‚9699 % _timestamp‚1.7858143288942573e+09 _runtime‚4074.084190334 _step‚7722”:ûÎÅÈ ' lr-AdamW/pg1‚0.00024996443672098385 ' lr-AdamW/pg2‚0.00024996443672098385 trainer/global_step‚9699 % _timestamp‚1.7858143288942573e+09 _runtime‚4074.084190334 _step‚772„³^¥c2Æ- _wandbruntime‚4074  _runtime‚4074 Ú^Ö'ÇÐO F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 166880372 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38191297 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚175680397573632 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8534846624256 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚162384233220832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8513958796972 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚175680561540096 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 102438693 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27783614 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚166914882198528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚65974911 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚166522126818304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8765679341568 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚4074.088633026 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170898174978444 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚162384216181472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 102438693 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚9699 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38191295 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8534846624256 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚166522126818304 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚175056973442560 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚773 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 102429037 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64441679 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 166870714 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚64441677 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64441679 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175056973442560 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64441677 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚65974661 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8765540278784 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚175057077911552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7858143288951583e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8534934053888 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8534934053888 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 102429037 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170898277820928 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8514044600096 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚175057077911552 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚166522143857664 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175056973442560 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64441677 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚65974661 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8765540278784 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚175057077911552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7858143288951583e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8534934053888 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8534934053888 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 102429037 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170898277820928 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8514044600096 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚175057077911552 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚166522143857664 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚175680561540096 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 102438693 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27783614 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚166914882198528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚65974911 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚166522126818304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8765679341568 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚4074.088633026 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170898174978444 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚162384216181472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 102438693 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚9699 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38191295 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8534846624256 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚166522126818304 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚175056973442560 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚773 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 102429037 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64441679 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 166870714 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚64441677 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856…Vò(Ø2É- _wandbruntime‚4074  _runtime‚4074'¿3•ëÊÚM >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚166539293581312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64448320 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚175075124301312 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 166897552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚166539310620672 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38195232 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 166897552 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65981440 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚230 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚4074.432215284 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8535726243840 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 102439560 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 166887880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚9699 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚166539310620672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 % _timestamp‚1.7858143292419176e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚162400956353792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64448322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64448322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚170915792623986 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27786438 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚166932064411648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0°5¶å&ËßM REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27786208 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚175075019825152 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚166539293581312 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 166887880 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65981672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 102439560 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 166887880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚9699 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚166539310620672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 % _timestamp‚1.7858143292419176e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚162400956353792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64448322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64448322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚170915792623986 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27786438 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚166932064411648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚166539293581312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64448320 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚175075124301312 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 166897552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚166539310620672 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38195232 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 166897552 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65981440 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚230 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚4074.432215284 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8535726243840 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175959307728384 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27927141 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27926876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8557933063604 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 $ _timestamp‚1.785814350013955e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 trainer/global_step‚9749 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 102957039 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64773829 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 102966695 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8578930189312 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚167380482008064 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚175959412197376 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚176585907341824 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚163221224797472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8810833269760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚167380482008064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚267 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38388145 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚171779072057952KcØÔO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚175959412197376 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚176585907341824 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚163221224797472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8810833269760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚167380482008064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚267 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38388145 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚171779072057952 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚66315288 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚66315021 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚176586071308288 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175959307728384 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27927141 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27926876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8557933063604 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 $ _timestamp‚1.785814350013955e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚265 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 trainer/global_step‚9749 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 102957039 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64773829 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 102966695 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8578930189312 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚167380482008064‰Å»2Ù- _wandbruntime‚4095  _runtime‚4095:gѨß&ÚÙM E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚167397648771072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 167757704 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚66321821 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8811734157824 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 102967562 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 167748032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64780472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚167792445155328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8579809816064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 167757704 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚778 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚175977354110976 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64780470 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚176604017451008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚163237982009152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38392082 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64780472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚167397648771072 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚167397631731712 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 102977232 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚163237964969792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8579809816064 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚242 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚9749 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8558724733702 $ _timestamp‚1.785814350363256e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚167792420251648 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚176604179313152 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64780470 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8811597199360 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64780470 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8811597199360 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚167397648771072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 167757704 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚66321821 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚244 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8811734157824 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 102967562 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 167748032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64780472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚167792445155328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8579809816064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 167757704 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚778 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚175977354110976 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64780470 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚176604017451008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚163237982009152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38392082 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64780472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚167397648771072 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚167397631731712 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 102977232 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚163237964969792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8579809816064 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚242 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚9749 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8558724733702 $ _timestamp‚1.785814350363256e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚167792420251648 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚176604179313152 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚177491415507456 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8601821527112 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 trainer/global_step‚9799 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28070076 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8622926324736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚168635593878528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚168238820158464 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚66655073 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚172659969137460 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚176861642014208 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 103485041 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚176861642014208 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65105977 _runtime‚4116.094701265 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8622838895104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65105979 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448•å´öÛ'æÕO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8601821527112 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 trainer/global_step‚9799 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28070076 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8622926324736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚168635593878528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚168238820158464 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚66655073 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚172659969137460 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚176861642014208 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 103485041 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚245 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚176861642014208 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65105977 _runtime‚4116.094701265 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8622838895104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65105979 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8855983498240 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚177491577376768 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 103494697 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚164058250452832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 168591018 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 103494697 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8855846532608 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚164058233413472 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 103485041 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38584997 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚168238820158464 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8622838895104 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚177491415507456 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 t¾”2ç- _wandbruntime‚4116  _runtime‚4116M<]¼à&èÚM \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8602699000446 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 168617856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 103505234 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8623718514688 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8856878789120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 168608184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28072685 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 168608184 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65112622 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 103505234 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚66661877 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38588932 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚66661617 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚9799 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚168255986921472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 103495564 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚168255969882112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8623718514688 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚177509522116608 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 % _timestamp‚1.7858143712548783e+09 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 168617856 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65112622 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65112620 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130ƒ™¬x8 éßM PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚66661617 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 trainer/global_step‚9799 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚168255986921472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 103495564 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚168255969882112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8623718514688 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚177509522116608 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 % _timestamp‚1.7858143712548783e+09 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 168617856 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65112622 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65112620 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8602699000446 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 168617856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 103505234 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8623718514688 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8856878789120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 168608184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28072685 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 168608184 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65112622 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 103505234 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚66661877 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38588932 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚177763976300032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚178396985933824 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚785 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28213744 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 104013043 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚178397143608832 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 104022699 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38781845 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚177764080769024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65438127 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚169097158308864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38781847 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7858143917179022e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 169451170 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65438129 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8666835030528 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚169495924814848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚164895242029472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚4136.911632016 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 104013043 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65438129 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28213997 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8645709990620 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚177764080769024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚177763976300032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680·Á’Û'ôÕO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚169097141269504 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8645624187496 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚164895259068832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 169460828 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚177763976300032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚178396985933824 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚785 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28213744 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 104013043 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚178397143608832 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 104022699 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38781845 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚177764080769024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65438127 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚169097158308864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38781847 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7858143917179022e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 169451170 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65438129 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8666835030528 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚169495924814848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚164895242029472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 _runtime‚4136.911632016 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 104013043 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65438129 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28213997 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8645709990620 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚177764080769024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚177763976300032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚169097158308864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚66995589 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8901061118976 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8666835030528 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 104022699 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚169495949718528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024‘9˜R 2õ- _wandbruntime‚4137  _runtime‚4137wlg¿öÚM REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38785782 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚169114308032512 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65444772 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚169513131931648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_t9²_9!#rain_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚169114325071872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _runtime‚4137.265212711 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 169468336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8902094467072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚173558586705106 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 104033236 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚177782022682624 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚178415251302400 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67002372 _step‚786 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65444770 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38785784 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚178415102023168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚164911999241152 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚169513156835328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚164911982201792 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 169478008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 104023566 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 169478008 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 104033236 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8646501660718 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65444770 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8667714650112 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚67002628 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚173558483862510 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 104023566 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28216590 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8646587463954 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚169114325071872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚169114308032512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8667802086912 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚9849FIâŒå&÷ßM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚169513156835328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚164911982201792 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 169478008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 104023566 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 169478008 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 104033236 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8646501660718 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65444770 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8667714650112 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚67002628 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚173558483862510 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 104023566 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28216590 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8646587463954 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚169114325071872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚169114308032512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 124375552 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8667802086912 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚9849 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38785782 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚169114308032512 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65444772 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚169513131931648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚169114325071872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _runtime‚4137.265212711 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 169468336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8902094467072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚173558586705106 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 104033236 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚177782022682624 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚178415251302400 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67002372 _step‚786 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65444770 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38785784 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚178415102023168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚164911999241152’»ls2ø- _wandbruntime‚4137  _runtime‚4137·+Sù‚ .loss/FRt1/detection_loss‚2.346023230390115 /loss/FRt2/detection_loss‚2.8051259142078764 0loss/FRt2/assignment_loss‚2.7408361867682576 3loss/FBt1/assignment_loss‚0.0007919551617779538 trainer/global_step‚9849 _runtime‚4137.266037449 2loss/SRqqt1/assignment_loss‚0.7996775234988245 0loss/SRqqt1/detection_loss‚1.003669358980264 1loss/SRqqt2/detection_loss‚1.8822806649841368 /loss/FBt2/detection_loss‚1.4552366390966602 &loss/total_loss‚17.705519028899083 epoch‚0 % _timestamp‚1.7858143920761456e+09 _step‚787 0loss/FRt1/assignment_loss‚2.4462838768959045 2loss/SRqqt2/assignment_loss‚0.9287856723894947 2loss/FBt2/assignment_loss‚3.31136194612706e-07 .loss/FBt1/detection_loss‚1.296807675389573VG¡§ ú‡ .loss/FRt1/detection_loss‚2.346023230390115 /loss/FRt2/detection_loss‚2.8051259142078764 0loss/FRt2/assignment_loss‚2.7408361867682576 3loss/FBt1/assignment_loss‚0.0007919551617779538 trainer/global_step‚9849 _runtime‚4137.266037449 2loss/SRqqt1/assignment_loss‚0.7996775234988245 0loss/SRqqt1/detection_loss‚1.003669358980264 1loss/SRqqt2/detection_loss‚1.8822806649841368 /loss/FBt2/detection_loss‚1.4552366390966602 &loss/total_loss‚17.705519028899083 epoch‚0 % _timestamp‚1.7858143920761456e+09 _step‚787 0loss/FRt1/assignment_loss‚2.4462838768959045 2loss/SRqqt2/assignment_loss‚0.9287856723894947 2loss/FBt2/assignment_loss‚3.31136194612706e-07 .loss/FBt1/detection_loss‚1.296807675389573“̓Õû‚(:’ úºÅÓÁµÇƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚131.172' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.724000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0-Ä^Qoü‚(:ä  úºÅÓú¯Ñƒ network.recv‚598446 disk.drbd1510.out‚0 disk.drbd1289.out‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd2041.out‚0 disk.sda.out‚ 174.796875 disk.drbd1550.in‚0 network.sent‚ 9.170659e+06 disk.drbd2041.in‚0 disk.drbd1965.out‚0) proc.memory.percent‚17.17204451560974 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.drbd2078.in‚0 disk.drbd1828.in‚0$ proc.memory.rssMB‚11253.87109375 disk.sda1.out‚0 disk.drbd1817.in‚0" disk.nvme0n1.out‚ 3944.37109375 cpu‚13.740455883197864 disk.drbd1938.out‚0 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.drbd2078.out‚0 disk.dm-0.out‚ 3944.37109375 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0$ memory_percent‚41.24532341957092* disk./.usagePercent‚32.959464385141914 disk.drbd1289.in‚0 disk.drbd1356.out‚0* proc.memory.availableMB‚38505.46484375 disk.drbd1510.in‚0 disk.drbd1584.out‚0 disk.drbd1285.in‚0 disk.dm-0.in‚ 440.05859375 disk.drbd2111.in‚0 disk.drbd1828.out‚0 disk.drbd2118.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1226.in‚0$ disk./.usageGB‚589.3486976623535 disk.sda2.out‚ 174.796875 disk.drbd1550.out‚0 proc.cpu.threads‚29 disk.sda2.in‚ 11.6328125 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.drbd1931.in‚0 disk.drbd1226.out‚0 disk.drbd1673.out‚0 disk.rbd1.out‚14.57275390625 disk.rbd0.out‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.drbd1285.out‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0¦mÑGœý‚(:‘ ‰»ÅÓîÖ”ƒ gpu.0.gpu‚38.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚129.337' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚43.11233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ßGpþ‚(:å  ‰»ÅÓ÷ÓÙƒ network.recv‚600088$ disk./.usageGB‚589.3486976623535 disk.rbd0.in‚0 disk.dm-0.in‚ 440.05859375 disk.sda.out‚ 175.078125 disk.drbd1552.in‚0 disk.sda2.in‚ 11.6328125 disk.rbd1.out‚ 14.6357421875 disk.sda.in‚ 11.6328125 disk.drbd1285.in‚0 disk.drbd1931.in‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0# proc.memory.rssMB‚ 11253.8828125* proc.memory.percent‚17.172062397003174 disk.drbd1584.in‚0 disk.drbd1289.in‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.rbd0.out‚0 disk.drbd1673.out‚0 disk.drbd1938.out‚0* disk./.usagePercent‚32.959464385141914 disk.drbd1356.in‚0 disk.drbd1226.out‚0 disk.drbd2078.in‚0 proc.cpu.threads‚29 disk.drbd1931.out‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 175.078125 disk.rbd1.in‚0 disk.drbd2041.out‚0$ memory_percent‚41.24539494514465 network.sent‚ 9.215741e+06 disk.drbd2041.in‚0 disk.drbd1550.out‚0 disk.drbd1285.out‚0 disk.drbd1226.in‚0 disk.drbd1965.out‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1552.out‚0 disk.drbd1828.in‚0 disk.dm-0.out‚3944.615234375 disk.drbd2111.out‚0 disk.drbd2118.in‚0 disk.nvme0n1.in‚ 440.05859375* proc.memory.availableMB‚38505.41796875 disk.drbd1817.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2118.out‚0# disk.nvme0n1.out‚3944.615234375 disk.drbd1584.out‚0 disk.drbd1550.in‚0 disk.drbd2078.out‚0 disk.drbd1673.in‚0 cpu‚13.740222650405522É2ÿ- _wandbruntime‚4157  _runtime‚4157«†Æ€À trainer/global_step‚9899 $ _timestamp‚1.785814412659775e+09 _runtime‚4157.849726359 _step‚788 & lr-AdamW/pg1‚0.0002551188740180451 & lr-AdamW/pg2‚0.0002551188740180451å&šTËÅ _runtime‚4157.849726359 _step‚788 & lr-AdamW/pg1‚0.0002551188740180451 & lr-AdamW/pg2‚0.0002551188740180451 trainer/global_step‚9899 $ _timestamp‚1.785814412659775e+09”¯3÷2‚- _wandbruntime‚4157  _runtime‚4157&+:DƒÏO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚179302575224832 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚170356305558528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65770277 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚169955496459264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚169955479419904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚178666415054848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚165732250645472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8689512651004 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8710831165952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 170320980 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 104541045 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8710918595584 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28357859 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8946427341312 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8710918595584 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8710831165952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚179302732899840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚174421866138960 _step‚789 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚169955479419904 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38978695 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚174421763296476 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚178666310585856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 104550701 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _runtime‚ 4157.85462382 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitoÉ6ï‘ r.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚67336556 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8689598454128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28357610 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚178666310585856 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 104550701 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚67336305 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚165732267684832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8946294569984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65770279(G¹Ú'„ÔO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚170356280654848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 170311322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚178666310585856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 104550701 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _runtime‚ 4157.85462382 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚67336556 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8689598454128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28357610 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚178666310585856 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 104550701 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚67336305 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚165732267684832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8946294569984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65770279 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚179302575224832 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚170356305558528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65770277 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚169955496459264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚169955479419904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚178666415054848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚165732250645472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8689512651004 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8710831165952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 170320980 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 104541045 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8710918595584 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28357859 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8946427341312 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8710918595584 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8710831165952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚179302732899840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚174421866138960 _step‚789 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚169955479419904 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38978695 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚174421763296476 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8711798222336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 104561238 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚178684356968448 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67343063 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚170373512675328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 170338160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚178684461444608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚170373487771648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65776920 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚179320843821056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚178684461444608 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28360431 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚165748990817792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚169972663222272 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8711710785536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚174439483784614 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8711710785536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179320684056064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 104551568 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 170338160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚28360675 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚165749007857152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38982634 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 104561238 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚67343309 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚174439380942018 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65776920 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 170328488 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8690475927462 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚4158.201120947 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚09m2%a&‡ÞM L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 170328488 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8711798222336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 104561238 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚178684356968448 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67343063 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚170373512675328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 170338160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚178684461444608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚170373487771648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65776920 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚179320843821056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚178684461444608 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28360431 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚165748990817792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚169972663222272 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8711710785536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚174439483784614 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8711710785536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179320684056064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 104551568 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 170338160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚28360675 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚165749007857152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38982634 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 104561238 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚67343309 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚174439380942018 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65776920 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 170328488 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8690475927462 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚4158.201120947 LCDeviceStatsMonitor.on_tra•gƒin_batch_end/allocation.small_pool.current‚9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0–fV52ˆ- _wandbruntime‚4158  _runtime‚4158Lóì^ ‰ƒ 0loss/FRt1/assignment_loss‚2.0901077911257744 2loss/SRqqt2/assignment_loss‚0.8781303672162308 3loss/FBt2/assignment_loss‚5.960460214282648e-08 .loss/FRt1/detection_loss‚2.482164392946288 epoch‚0 trainer/global_step‚9899 _step‚791 2loss/SRqqt1/assignment_loss‚0.7584165093432972 3loss/FBt1/assignment_loss‚0.0011124175453601094 /loss/FRt2/detection_loss‚2.5245659236448357 .loss/FBt1/detection_loss‚2.104396010438601 &loss/total_loss‚17.997427179300132 0loss/FRt2/assignment_loss‚3.1927907711052033 1loss/SRqqt1/detection_loss‚1.1416270907099981 /loss/FBt2/detection_loss‚0.9566643602253558 _runtime‚4158.201811349 0loss/SRqqt2/detection_loss‚1.867451485394585 % _timestamp‚1.7858144130120463e+09HwïðŠˆ 0loss/FRt1/assignment_loss‚2.0901077911257744 2loss/SRqqt2/assignment_loss‚0.8781303672162308 3loss/FBt2/assignment_loss‚5.960460214282648e-08 .loss/FRt1/detection_loss‚2.482164392946288 epoch‚0 trainer/global_step‚9899 _step‚791 2loss/SRqqt1/assignment_loss‚0.7584165093432972 3loss/FBt1/assignment_loss‚0.0011124175453601094 /loss/FRt2/detection_loss‚2.5245659236448357 .loss/FBt1/detection_loss‚2.104396010438601 &loss/total_loss‚17.997427179300132 0loss/FRt2/assignment_loss‚3.1927907711052033 1loss/SRqqt1/detection_loss‚1.1416270907099981 /loss/FBt2/detection_loss‚0.9566643602253558 _runtime‚4158.201811349 0loss/SRqqt2/detection_loss‚1.867451485394585 % _timestamp‚1.7858144130120463e+09—ƒþ:ë‹‚(:’ ˜»ÅÓ Ûƒ gpu.0.gpu‚31.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚121.798' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚40.599333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¹œsŒ‚(:è  ˜»ÅÓ·šØƒ# proc.memory.rssMB‚ 11253.8828125 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.sda2.in‚ 11.6328125 proc.cpu.threads‚29 disk.drbd1552.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1289.out‚0 disk.drbd1584.out‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0 disk.sda1.out‚0 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.rbd0.in‚0 disk.drbd1226.out‚0 cpu‚13.739873490714967 disk.drbd2041.out‚0 disk.drbd1965.in‚0 disk.dm-0.in‚ 440.05859375 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.sda1.in‚0 disk.drbd2041.in‚0 disk.drbd1289.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 14.6357421875# disk.nvme0n1.out‚3944.888671875 disk.sda.out‚ 175.5078125 disk.drbd1550.in‚0 disk.rbd0.out‚0 disk.drbd1285.in‚0 network.sent‚ 9.218786e+06$ disk./.usageGB‚589.3486976623535 disk.drbd1817.in‚0 disk.drbd1356.out‚0% memory_percent‚41.245871782302856* proc.memory.availableMB‚38505.10546875 disk.nvme0n1.in‚ 440.05859375 disk.dm-0.out‚3944.888671875 network.recv‚601076 disk.drbd1817.out‚0 disk.drbd1931.in‚0 disk.drbd1550.out‚0 disk.sda.in‚ 11.6328125 disk.drbd2078.in‚0* disk./.usagePercent‚32.959464385141914 disk.sda2.out‚ 175.5078125 disk.drbd1828.in‚0 disk.drbd1828.out‚0* proc.memory.percent‚17.172062397003174 disk.drbd1584.in‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.drbd2118.out‚0 disk.drbd1510.out‚0Õ§ d2- _wandbruntime‚4179  _runtime‚4179üZ»PÈŽÂ ' lr-AdamW/pg1‚0.00025640748334231035 ' lr-AdamW/pg2‚0.00025640748334231035 trainer/global_step‚9949 % _timestamp‚1.7858144339423218e+09 _runtime‚ 4179.13362033 _step‚792[›ÍÇ % _timestamp‚1.7858144339423218e+09 _runtime‚ 4179.13362033 _step‚792 ' lr-AdamW/pg1‚0.00025640748334231035 ' lr-AdamW/pg2‚0.00025640748334231035 trainer/global_step‚9949˜*ÿ‚2- _wandbruntime‚4179  _runtime‚4179Z)LÖ'‘ÐO @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39175545 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚171216636494848 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 105078703 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚175302660375984 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 105078703 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚67677406 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚67677669 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 171171474 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8754914731008 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8754914731008 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚166569276300832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚166569259261472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8733401114512 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚170813817570304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8754827301376 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚263 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66102427 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 171181132 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66102429 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 105069047 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28502122 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚175302763218468 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 105069047 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8991657529856 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39175547 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚261 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚170813817570304 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚179568644871680 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚179568644871680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8991522661376 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚170813834609664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8733486917636 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚179568749340672 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66102427 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28501861 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _runtime‚4179.133696619 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚180208318928384 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170813834609664 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8754827301376 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚179568749340672 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130¦”6 Û'’ÕO WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8754827301376 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚179568749340672 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39175545 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚171216636494848 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 105078703 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚175302660375984 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 105078703 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚67677406 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚67677669 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 171171474 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8754914731008 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8754914731008 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚166569276300832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚166569259261472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8733401114512 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚170813817570304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8754827301376 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚263 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66102427 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 171181132 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66102429 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 105069047 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28502122 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚175302763218468 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 105069047 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8991657529856 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39175547 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚261 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚170813817570304 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚179568644871680 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚179568644871680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8991522661376 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚170813834609664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8733486917636 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚179568749340672 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66102427 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28501861 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 _runtime‚4179.133696619 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚180208318928384 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170813834609664™^øfÈ2“- _wandbruntime‚4179  _runtime‚4179–ô&Šæ”ÚM E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚166585999433792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚4179.487982708 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚179586691254272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚171233843611648 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 105089240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚170830984333312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚175320380864122 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8755706920960 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 171198312 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8734278587734 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚179586795730432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚170831001372672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 105089240 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚9949 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39179482 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8755706920960 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8992437054976 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67684261 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28504779 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _step‚794 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 171188640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8992571916288 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚166586016473152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66109070 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚170830984333312 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66109070 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7858144342976909e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8755794357760 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 105079570 VGDeviceStatëá½^úsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8755794357760 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 171198312 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8734364390970 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 105079570 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39179484 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚180226440431616ûøéå&•ßM OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 105089240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚170830984333312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚175320380864122 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8755706920960 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 171198312 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8734278587734 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚179586795730432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚170831001372672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 105089240 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚9949 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39179482 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8755706920960 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8992437054976 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67684261 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28504779 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 _step‚794 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 171188640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8992571916288 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚166586016473152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66109070 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚170830984333312 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66109070 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7858144342976909e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8755794357760 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 105079570 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8755794357760 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚250 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 171198312 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8734364390970 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 105079570 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39179484 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚248 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚180226440431616 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚166585999433792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚4179.487982708 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚179586691254272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚171233843611648 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251šÂaÍ2–- _wandbruntime‚4179  _runtime‚4179›Ø$¯— /loss/FRt2/detection_loss‚2.5205536996140894 1loss/SRqqt1/detection_loss‚0.9501874780550293 .loss/FRt1/detection_loss‚2.291310603362183 1loss/SRqqt2/detection_loss‚1.5174769703298807 _runtime‚4179.488667218 _step‚795 2loss/SRqqt1/assignment_loss‚0.8801223875975479 1loss/FBt1/assignment_loss‚0.05249148599564724 /loss/FBt1/detection_loss‚1.3416946249703565 epoch‚0 % _timestamp‚1.7858144342988946e+09 .loss/FRt1/assignment_loss‚1.99631747464801 0loss/FRt2/assignment_loss‚2.9843938799551974 2loss/SRqqt2/assignment_loss‚0.6429881849590774 3loss/FBt2/assignment_loss‚0.0025762927073630457 /loss/FBt2/detection_loss‚0.8797243087767408 &loss/total_loss‚16.059837390971122 trainer/global_step‚9949ÑÔXK ˜† /loss/FRt2/detection_loss‚2.5205536996140894 1loss/SRqqt1/detection_loss‚0.9501874780550293 .loss/FRt1/detection_loss‚2.291310603362183 1loss/SRqqt2/detection_loss‚1.5174769703298807 _runtime‚4179.488667218 _step‚795 2loss/SRqqt1/assignment_loss‚0.8801223875975479 1loss/FBt1/assignment_loss‚0.05249148599564724 /loss/FBt1/detection_loss‚1.3416946249703565 epoch‚0 % _timestamp‚1.7858144342988946e+09 .loss/FRt1/assignment_loss‚1.99631747464801 0loss/FRt2/assignment_loss‚2.9843938799551974 2loss/SRqqt2/assignment_loss‚0.6429881849590774 3loss/FBt2/assignment_loss‚0.0025762927073630457 /loss/FBt2/detection_loss‚0.8797243087767408 &loss/total_loss‚16.059837390971122 trainer/global_step‚9949›¸k¸œ™‚(:‘ §»ÅÓÀú¬ƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.635' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚32.211666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0;—hš‚(:Ý  §»ÅÓØ–Ôƒ disk.drbd1285.in‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.rbd1.out‚ 14.6357421875 disk.drbd1931.in‚0 disk.drbd2041.out‚0$ memory_percent‚41.24577045440674 disk.sda2.out‚175.75 disk.drbd1584.out‚0* proc.memory.percent‚17.172062397003174 proc.cpu.threads‚29" disk.dm-0.out‚3945.08056640625 disk.drbd1289.out‚0 disk.sda2.in‚ 11.6328125 cpu‚13.739644694559386 disk.drbd1817.out‚0* disk./.usagePercent‚32.959464385141914 disk.drbd1510.out‚0% disk.nvme0n1.out‚3945.08056640625 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.drbd1356.out‚0 disk.drbd1510.in‚0 disk.drbd1931.out‚0 disk.sda.in‚ 11.6328125 disk.drbd1226.in‚0 disk.drbd1673.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚ 440.05859375 disk.sda.out‚175.75 disk.drbd2078.out‚0 disk.drbd1828.in‚0 disk.drbd1226.out‚0$ disk./.usageGB‚589.3486976623535 disk.rbd0.out‚0 disk.drbd1552.out‚0 disk.drbd1938.in‚0 network.sent‚ 9.2638e+06 disk.rbd0.in‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1584.in‚0 disk.drbd1965.out‚0 disk.drbd1289.in‚0 disk.sda1.in‚0 disk.drbd1965.in‚0( proc.memory.availableMB‚ 38505.171875 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.drbd2041.in‚0 disk.drbd1550.out‚0 disk.drbd1285.out‚0 disk.sda1.out‚0# proc.memory.rssMB‚ 11253.8828125 disk.drbd2118.in‚0 disk.drbd1356.in‚0 disk.drbd1550.in‚0 network.recv‚602526 disk.drbd2078.in‚0 disk.rbd1.in‚0~´Í$2›- _wandbruntime‚4199  _runtime‚4199œ‹,Çɜà ' lr-AdamW/pg2‚0.00025769609266657566 trainer/global_step‚9999 % _timestamp‚1.7858144544643252e+09 _runtime‚4199.655633085 _step‚796 ' lr-AdamW/pg1‚0.00025769609266657566N“.>ÎÈ ' lr-AdamW/pg1‚0.00025769609266657566 ' lr-AdamW/pg2‚0.00025769609266657566 trainer/global_step‚9999 % _timestamp‚1.7858144544643252e+09 _runtime‚4199.655633085 _step‚796œâ-fû2ž- _wandbruntime‚4199  _runtime‚4199±aÎÖ'ŸÐO =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 105597049 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66434577 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚68017245 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39372395 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28644848 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28644590 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚797 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚68016985 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚9999 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 % _timestamp‚1.7858144544650037e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66434579 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _runtime‚4199.655743523 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8798910866432 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚180470979157504 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 105606705 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 172031626 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8777289578020 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚180470979157504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39372397 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚172077017238528 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚180471083626496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8798910866432 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚180471083626496 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚176183660297976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66434577 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8798823436800 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚181113884122112 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936Lˆ:¬ ÕO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚180470979157504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39372397 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚172077017238528 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚180471083626496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8798910866432 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚180471083626496 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚176183660297976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66434577 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8798823436800 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚181113884122112 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 105597049 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66434577 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚68017245 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39372395 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28644848 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28644590 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚797 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚68016985 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.smalˆpF”/l_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚9999 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 % _timestamp‚1.7858144544650037e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66434579 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _runtime‚4199.655743523 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8798910866432 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚180470979157504 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 105606705 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 172031626 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚258 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166063616 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8777289578020 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚260 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 172058464 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39376334 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66441222 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 105607572 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 105607572 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 172048792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚167423025089152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8799703056384 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚180489025540096 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28647437 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66441220 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚68024021 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9037761420800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚798 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39376332 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 172058464 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚181131985776128 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚172094199451648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚167423008049792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚176201277943630 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66441220 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚4200.004730435 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚180489130016256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 # _timestamp‚1.78581445481441e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8778252854478 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚9999 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 105617242 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚181131830205440 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66441222 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8799790493184 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68023769 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚172094224355328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚171689322483712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8799703056384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚176201277943630 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66441220 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚4200.004730435 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚180489130016256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 # _timestamp‚1.78581445481441e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8778252854478 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚9999 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 105617242 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚181131830205440 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66441222 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8799790493184 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68023769 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚172094224355328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚171689322483712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8799703056384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 172058464 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39376334 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66441222 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 105607572 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 105607572 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 172048792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚167423025089152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8799703056384 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚180489025540096 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28647437 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66441220 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚68024021 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9037761420800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚798 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39376332 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 172058464 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚181131985776128 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚252 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚172094199451648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚167423008049792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66766729 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 106134707 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 _runtime‚4220.822645277 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚172937373078528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66766727 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚172530493871104 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9082056079872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚801 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 106125051 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8842819572224 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39569247 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚172937348174848 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39569245 WJDeviceStats=àù¸Monitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚181373417912320 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8842907001856 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 106125051 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8821178041528 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28788794 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚177064454535000 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚181373417912320 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚177064557377484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚66766729 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28788534 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8821263844652 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 $ _timestamp‚1.785814475628769e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66766727 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚182019267289088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 106134707 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 172901436 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚168243293532832 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8842819572224 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚172530510910464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182019429158400 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚181373313443328 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚172530510910464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚68357779 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9081919114240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚181373313443328 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚168243276493472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130F÷DÛ'°ÕO F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚66766729 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28788534 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8821263844652 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 $ _timestamp‚1.785814475628769e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66766727 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚182019267289088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 106134707 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 172901436 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚168243293532832 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8842819572224 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚172530510910464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182019429158400 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚181373313443328 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚172530510910464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚68357779 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9081919114240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚181373313443328 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚168243276493472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66766729 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 106134707 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 _runtime‚4220.822645277 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚262 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚172937373078528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66766727 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚172530493871104 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9082056079872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 _step‚801 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 106125051 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8842819572224 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39569247 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚172937348174848 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39569245 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚181373417912320 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8842907001856 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 106125051 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8821178041528 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28788794 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚177064454535000 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚181373417912320 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚177064557377484¡{ ÿ2±- _wandbruntime‚4221  _runtime‚4221‡íá&²ÛM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8843786628608 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚182037377553920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚4221.181317598 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66773370 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858144759910016e+09 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚172954580195328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9082950832128 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 106135574 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39573182 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66773370 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚172547677673472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8843699191808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 172908944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68364563 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 106135574 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚172954555291648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚177082175023138 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 trainer/global_step‚10049 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28791381 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚181391359825920 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚802 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚172547660634112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66773372 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8822141317986 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 172918616 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 172908944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8843786628608 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 106145244 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66773372 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 172918616 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚177082072180542 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚168260033705152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392h¶»p7³àM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚172954580195328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9082950832128 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 106135574 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39573182 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66773370 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚172547677673472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8843699191808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 172908944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68364563 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 106135574 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚172954555291648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚177082175023138 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 trainer/global_step‚10049 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28791381 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚181391359825920 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 _step‚802 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚172547660634112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66773372 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8822141317986 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 172918616 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 172908944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8843786628608 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 106145244 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66773372 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 172918616 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚177082072180542 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚168260033705152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8843786628608 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚182037377553920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚4221.181317598 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66773370 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 % _timestamp‚1.7858144759910016e+09 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856¢ç¹h`2´- _wandbruntime‚4221  _runtime‚4221ÈHâ µ„ _step‚803 .loss/FRt2/detection_loss‚2.724955055397004 1loss/SRqqt1/detection_loss‚0.9903778315873335 /loss/FBt1/detection_loss‚1.2327318550949846 &loss/total_loss‚16.857428571773855 trainer/global_step‚10049 _runtime‚4221.182388338 0loss/FRt1/assignment_loss‚2.1189806473882573 /loss/FRt1/detection_loss‚2.3051464061988027 epoch‚0 2loss/SRqqt2/assignment_loss‚0.7656919621406313 3loss/FBt2/assignment_loss‚6.947309455050533e-05 /loss/FBt2/detection_loss‚1.2094734757175218 % _timestamp‚1.7858144759922538e+09 /loss/FRt2/assignment_loss‚2.996788825839758 2loss/SRqqt1/assignment_loss‚0.9167977189182593 2loss/FBt1/assignment_loss‚5.06714600950465e-05 1loss/SRqqt2/detection_loss‚1.5963646489366585[´~e¶‰ .loss/FRt2/detection_loss‚2.724955055397004 1loss/SRqqt1/detection_loss‚0.9903778315873335 /loss/FBt1/detection_loss‚1.2327318550949846 &loss/total_loss‚16.857428571773855 trainer/global_step‚10049 _runtime‚4221.182388338 0loss/FRt1/assignment_loss‚2.1189806473882573 /loss/FRt1/detection_loss‚2.3051464061988027 epoch‚0 2loss/SRqqt2/assignment_loss‚0.7656919621406313 3loss/FBt2/assignment_loss‚6.947309455050533e-05 /loss/FBt2/detection_loss‚1.2094734757175218 % _timestamp‚1.7858144759922538e+09 /loss/FRt2/assignment_loss‚2.996788825839758 2loss/SRqqt1/assignment_loss‚0.9167977189182593 2loss/FBt1/assignment_loss‚5.06714600950465e-05 1loss/SRqqt2/detection_loss‚1.5963646489366585 _step‚803£Ä€†™·‚(:’ Ô»ÅÓÅ浃 gpu.0.gpu‚40.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚101.254' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚33.751333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚02ýé3s¸‚(:è  Ô»ÅÓšÁƒ disk.drbd2041.out‚0 disk.sda.in‚ 11.6328125 disk.drbd2078.in‚0 disk.drbd1356.out‚0 disk.sda.out‚ 177.18359375 disk.drbd1226.in‚0 disk.drbd1673.out‚0 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.drbd1938.in‚0 disk.drbd1550.in‚0 disk.drbd1285.in‚0 disk.drbd1931.in‚0) proc.memory.percent‚17.17209815979004 proc.cpu.threads‚29" disk.dm-0.out‚3947.13427734375 disk.drbd1552.in‚0( proc.memory.availableMB‚ 38505.046875 disk.rbd0.out‚0 disk.drbd1584.in‚0 disk.drbd1584.out‚0! proc.memory.rssMB‚ 11253.90625 disk.drbd1965.in‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.sda2.in‚ 11.6328125 disk.drbd2041.in‚0 disk.drbd1285.out‚0 disk.drbd2118.in‚0 network.sent‚ 9.356863e+06 disk.drbd1226.out‚0 disk.nvme0n1.in‚ 440.05859375 cpu‚13.738676883254707 disk.dm-0.in‚ 440.05859375 disk.drbd1289.out‚0 disk.drbd2078.out‚0 disk.drbd1289.in‚0 disk.rbd1.in‚0 disk.drbd1931.out‚0* disk./.usagePercent‚32.959444331384866 disk.sda1.out‚0 disk.sda2.out‚ 177.18359375 network.recv‚606599$ disk./.usageGB‚589.3483390808105 disk.drbd1965.out‚0 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.rbd1.out‚ 14.7216796875 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd1817.out‚0 disk.drbd1356.in‚0$ memory_percent‚41.24596118927002 disk.drbd1828.in‚0 disk.drbd1938.out‚0 disk.drbd2118.out‚0% disk.nvme0n1.out‚3947.13427734375%ïC¾2¹- _wandbruntime‚4241  _runtime‚4241 þGȺ & lr-AdamW/pg1‚0.0002602733113151063 & lr-AdamW/pg2‚0.0002602733113151063 trainer/global_step‚10099 % _timestamp‚1.7858144965593297e+09 _runtime‚4241.749274809 _step‚804ïˆíÍ»Ç % _timestamp‚1.7858144965593297e+09 _runtime‚4241.749274809 _step‚804 & lr-AdamW/pg1‚0.0002602733113151063 & lr-AdamW/pg2‚0.0002602733113151063 trainer/global_step‚10099¤¹vèa2¼- _wandbruntime‚4241  _runtime‚4241ï9ˆXÖ'½ÐO JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚68697432 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182924953978368 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67098879 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 trainer/global_step‚10099 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚177945351614508 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 _step‚805 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚169080285109472 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 173761588 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8886815707648 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 106653053 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39766097 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9127225059840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9127098579968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67098879 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67098877 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 106662709 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚234 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 173751930 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚173388832021504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8865152308160 $ _timestamp‚1.785814496560253e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚182275752198144 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚182275647729152 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28931571 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚169080285109472 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 173761588 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8886815707648 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 106653053 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39766097 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9127225059840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9127098579968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67098879 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67098877 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 106662709 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚234 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 173751930 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚173388832021504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 126479872 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8865152308160 $ _timestamp‚1.785814496560253e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚182275752198144 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚182275647729152 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28931571 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚68697432 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182924953978368 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67098879 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 151383552 trainer/global_step‚10099 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚177945351614508 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 _step‚805¥½Ñ2¿- _wandbruntime‚4242  _runtime‚4242( *›¸ ÀÙM cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚173814936035328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 _runtime‚4242.100627558 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8887782764032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 trainer/global_step‚10099 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚173405998784512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 173778768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚177963072102646 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68704206 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 173778768 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚182293798587904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67105522 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚182293798587904 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67105520 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 106663576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚182293694111744 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9127997203968 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_trai²í ª'n_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚182943070197760 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚177962969260050 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 # _timestamp‚1.78581449691032e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 173769096 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67105520 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚173814911131648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9128134162432 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 106663576 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68704206 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 173778768 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚182293798587904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67105522 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚182293798587904 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67105520 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 106663576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚182293694111744 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9127997203968 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚182943070197760 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚177962969260050 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 # _timestamp‚1.78581449691032e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 173769096 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67105520 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚173814911131648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9128134162432 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 106663576 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 173778768¦Îð@2Â- _wandbruntime‚4242  _runtime‚4242Ì–‹-À 0loss/SRqqt1/detection_loss‚1.074443040167292 $loss/total_loss‚16.4282562636983 % _timestamp‚1.7858144969115226e+09 _step‚807 /loss/FRt1/assignment_loss‚2.298317386748943 4loss/FBt1/assignment_loss‚0.00010769542830146861 4loss/FBt2/assignment_loss‚1.6114994212801793e-06 .loss/FBt1/detection_loss‚1.300125507105674 1loss/SRqqt1/assignment_loss‚0.837735492597955 1loss/SRqqt2/assignment_loss‚1.009114063921911 .loss/FRt2/detection_loss‚2.463912890588536 /loss/FBt2/detection_loss‚0.9164611287415028 epoch‚0 _runtime‚4242.101301081 /loss/FRt2/assignment_loss‚2.853486943946165 /loss/FRt1/detection_loss‚2.3130230602431805 1loss/SRqqt2/detection_loss‚1.3615274427094213 trainer/global_step‚10099€RÔ Ä… 0loss/SRqqt1/detection_loss‚1.074443040167292 $loss/total_loss‚16.4282562636983 % _timestamp‚1.7858144969115226e+09 _step‚807 /loss/FRt1/assignment_loss‚2.298317386748943 4loss/FBt1/assignment_loss‚0.00010769542830146861 4loss/FBt2/assignment_loss‚1.6114994212801793e-06 .loss/FBt1/detection_loss‚1.300125507105674 1loss/SRqqt1/assignment_loss‚0.837735492597955 1loss/SRqqt2/assignment_loss‚1.009114063921911 .loss/FRt2/detection_loss‚2.463912890588536 /loss/FBt2/detection_loss‚0.9164611287415028 epoch‚0 _runtime‚4242.101301081 /loss/FRt2/assignment_loss‚2.853486943946165 /loss/FRt1/detection_loss‚2.3130230602431805 1loss/SRqqt2/detection_loss‚1.3615274427094213 trainer/global_step‚10099§'YÔðœÅ‚(:‘ ã»ÅÓúŸ¿ƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚120.565' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.18833333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0D-GOjÆ‚(:ß  ã»ÅÓܧìƒ disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.drbd2111.in‚0 disk.drbd1289.in‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1356.out‚0 disk.drbd1931.in‚0 disk.drbd1552.out‚0 disk.drbd1965.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2041.out‚0! disk.dm-0.out‚3948.2294921875 disk.rbd1.in‚0 disk.rbd1.out‚ 14.7841796875) proc.memory.percent‚17.17209815979004 disk.rbd0.in‚0 disk.dm-0.in‚ 440.05859375 disk.drbd1938.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 11.6328125! proc.memory.rssMB‚ 11253.90625 disk.drbd1817.in‚0( proc.memory.availableMB‚ 38505.015625 disk.drbd1510.in‚0 disk.drbd1510.out‚0$ disk./.usageGB‚589.3482704162598 disk.drbd1584.out‚0 disk.drbd2118.in‚0$ disk.nvme0n1.out‚3948.2294921875 proc.cpu.threads‚29 disk.drbd2041.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0$ memory_percent‚41.24600887298584 disk.drbd1285.in‚0 network.recv‚608205 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.sda.in‚ 11.6328125 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.drbd1673.out‚0 cpu‚13.738307821014756 network.sent‚ 9.401888e+06 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.drbd1356.in‚0 disk.sda.out‚ 177.59375) disk./.usagePercent‚32.95944049130374 disk.drbd1550.in‚0 disk.drbd1673.in‚0 disk.sda2.out‚ 177.59375~4§R‘Ç‚(:† ò»ÅÓÌ¢‚ƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚117.576' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚39.192 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0?¤woÈ‚(:ä  ò»ÅÓ¡ÖɃ disk.drbd1931.out‚0 disk.sda.out‚ 178.328125 disk.drbd1673.out‚0 disk.drbd1289.out‚0 disk.drbd1938.out‚0 disk.drbd1584.out‚0 disk.drbd1938.in‚0 cpu‚13.738003214772885 disk.sda2.in‚ 11.6328125% disk.nvme0n1.out‚3948.47705078125 disk.drbd2111.out‚0 proc.cpu.threads‚29 disk.drbd1931.in‚0 disk.sda.in‚ 11.6328125 disk.rbd0.out‚0 disk.drbd2111.in‚0 disk.drbd2118.out‚0) proc.memory.percent‚17.17219352722168 disk.drbd1673.in‚0$ memory_percent‚41.24576449394226* proc.memory.availableMB‚38505.17578125 disk.rbd1.out‚ 14.7841796875 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.drbd2078.out‚0 disk.dm-0.in‚ 440.05859375 disk.sda1.in‚0 disk.drbd1550.in‚0 disk.sda1.out‚0 disk.drbd1552.out‚0) disk./.usagePercent‚32.95944198466862$ disk./.usageGB‚589.3482971191406 disk.drbd1828.out‚0 disk.drbd1510.out‚0 network.sent‚ 9.44693e+06 network.recv‚609778 disk.drbd1356.in‚0 disk.drbd1584.in‚0 disk.drbd1285.in‚0 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0" disk.dm-0.out‚3948.47705078125 disk.drbd1817.in‚0 disk.drbd1550.out‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.sda2.out‚ 178.328125 disk.drbd2041.in‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1226.out‚0 disk.drbd1552.in‚0! proc.memory.rssMB‚ 11253.96875 disk.rbd1.in‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd2118.in‚0[‰Ì#2É- _wandbruntime‚4262  _runtime‚4262ÉzÈÊ _step‚808 & lr-AdamW/pg1‚0.0002615619206393716 & lr-AdamW/pg2‚0.0002615619206393716 trainer/global_step‚10149 % _timestamp‚1.7858145175458784e+09 _runtime‚4262.735829368—'¸bÍËÇ & lr-AdamW/pg1‚0.0002615619206393716 & lr-AdamW/pg2‚0.0002615619206393716 trainer/global_step‚10149 % _timestamp‚1.7858145175458784e+09 _runtime‚4262.735829368 _step‚808¨Çgü2Ì- _wandbruntime‚4262  _runtime‚4262[ጆQ%ÍÑO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚183177982014976 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚263 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29074737 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 107190711 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚183830347027968 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚169917310764832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67431029 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚183177982014976 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚174247187211264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚174247170171904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚174247170171904 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 107190711 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚69037947 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚183178086483968 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67431027 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚183830347027968 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚169917310764832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67431029 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚183177982014976 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚174247187211264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚174247170171904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚174247170171904 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 107190711 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚69037947 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚183178086483968 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67431027 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚263 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29074737 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 107190711 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0©clž¤2Ï- _wandbruntime‚4263  _runtime‚4263òÔ‰¯á&ÐÛM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8931778899456 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 174629248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚174675266971648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 174629248 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚169934033897792 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚174264353974272 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8931691462656 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174675291875328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚169934050937152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39966884 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 174638920 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67437670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29077875 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67437672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8931778899456 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9173323066880 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67437670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29077619 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚69044759 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 107201248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚178843866339558 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚183196028397568 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67437672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39966882 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚183848457274368 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚174264353974272 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 % _timestamp‚1.7858145179049103e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 174638920 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69044501 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚183848614942208 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚183196132873728 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 107201248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9173190302720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚183196132873728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8909918245002 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚10149Ã7næ&ÑàM NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67437670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29077875 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67437672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8931778899456 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9173323066880 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67437670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29077619 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚69044759 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 107201248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚178843866339558 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚183196028397568 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67437672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39966882 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚183848457274368 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚174264353974272 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 % _timestamp‚1.7858145179049103e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 174638920 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69044501 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚183848614942208 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚256 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚183196132873728 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 107201248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9173190302720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚183196132873728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8909918245002 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 trainer/global_step‚10149 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8931778899456 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 174629248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚174675266971648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 174629248 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚169934033897792 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚174264353974272 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8931691462656 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚258 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174675291875328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚169934050937152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39966884 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 174638920 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121ªœDmB2Ò- _wandbruntime‚4263  _runtime‚4263N`ù? Ó„ 0loss/FRt1/assignment_loss‚2.5802793807761613 0loss/SRqqt1/detection_loss‚1.322323237698692 1loss/SRqqt2/detection_loss‚1.1054874507070922 /loss/FBt1/detection_loss‚1.4809756310656668 % _timestamp‚1.7858145179061377e+09 0loss/FRt2/assignment_loss‚2.9488301450693153 2loss/SRqqt1/assignment_loss‚0.8890324089504125 2loss/SRqqt2/assignment_loss‚0.8450959403651641 4loss/FBt1/assignment_loss‚1.1574160810923217e-05 _runtime‚ 4263.0959357 4loss/FBt2/assignment_loss‚3.5762775496550604e-08 .loss/FRt1/detection_loss‚2.369419645344795 /loss/FRt2/detection_loss‚2.5748348183269743 /loss/FBt2/detection_loss‚1.3731075120158494 epoch‚0 _step‚811 %loss/total_loss‚17.48939778024371 trainer/global_step‚10149IœÅüÔ‰ 0loss/FRt1/assignment_loss‚2.5802793807761613 0loss/SRqqt1/detection_loss‚1.322323237698692 1loss/SRqqt2/detection_loss‚1.1054874507070922 /loss/FBt1/detection_loss‚1.4809756310656668 % _timestamp‚1.7858145179061377e+09 0loss/FRt2/assignment_loss‚2.9488301450693153 2loss/SRqqt1/assignment_loss‚0.8890324089504125 2loss/SRqqt2/assignment_loss‚0.8450959403651641 4loss/FBt1/assignment_loss‚1.1574160810923217e-05 _runtime‚ 4263.0959357 4loss/FBt2/assignment_loss‚3.5762775496550604e-08 .loss/FRt1/detection_loss‚2.369419645344795 /loss/FRt2/detection_loss‚2.5748348183269743 /loss/FBt2/detection_loss‚1.3731075120158494 epoch‚0 _step‚811 %loss/total_loss‚17.48939778024371 trainer/global_step‚10149«¶ZdnÕ‚(:… ¼ÅӪǦƒ gpu.0.gpu‚34.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚96.081' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚32.027 gpuÁDú9u.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡ÝÒpÖ‚(:å  ¼ÅÓÔñ¾ƒ disk.drbd1552.out‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.sda.out‚ 178.5859375 disk.drbd1356.in‚0% memory_percent‚41.246211528778076! proc.memory.rssMB‚ 11253.96875$ disk./.usageGB‚589.3482780456543 disk.rbd0.in‚0 proc.cpu.threads‚29 disk.drbd2041.in‚0 disk.dm-0.in‚ 440.05859375 disk.drbd1938.out‚0 disk.drbd1673.in‚0 disk.drbd1931.out‚0 cpu‚13.737584224922442 disk.drbd1552.in‚0 disk.drbd1817.in‚0 disk.drbd1584.out‚0 disk.drbd2078.in‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.sda2.in‚ 11.6328125 disk.rbd1.out‚ 14.81640625 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.sda1.in‚0 disk.sda1.out‚0% disk.nvme0n1.out‚3948.84912109375 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1817.out‚0 disk.drbd1965.out‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1510.out‚0 disk.drbd1673.out‚0 disk.sda.in‚ 11.6328125 disk.drbd1550.out‚0) proc.memory.availableMB‚ 38504.8828125) proc.memory.percent‚17.17219352722168 network.recv‚610713 disk.drbd1226.out‚0 disk.drbd1285.in‚0" disk.dm-0.out‚3948.84912109375 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1356.out‚0* disk./.usagePercent‚32.959440917979414 disk.drbd1965.in‚0 disk.drbd1289.in‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.sda2.out‚ 178.5859375 disk.drbd1584.in‚0 network.sent‚ 9.44996e+06 disk.drbd1226.in‚0 disk.drbd2041.out‚0>üŠÐ2×- _wandbruntime‚4283  _runtime‚4283õô8lÈØ _step‚812 & lr-AdamW/pg1‚0.0002628505299636369 & lr-AdamW/pg2‚0.0002628505299636369 trainer/global_step‚10199 % _timestamp‚1.7858145386824758e+09 _runtime‚4283.872405971ªI£ÍÙÇ trainer/global_step‚10199 % _timestamp‚1.7858145386824758e+09 _runtime‚4283.872405971 _step‚812 & lr-AdamW/pg1‚0.0002628505299636369 & lr-AdamW/pg2‚0.0002628505299636369¬¼Æ®2Ú- _wandbruntime‚4283  _runtime‚4283Y~×'ÛÑO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 107709057 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚69378450 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚175518415694848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 175481892 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚10199 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 107718713 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8952929235176 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40159795 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184080316300800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚175105508322304 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9217666345472 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 107709057 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚175518440598528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚179707145773524 % _timestamp‚1.7858145386834567e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚175105525361664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9217529379840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8974895408128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚184736106944000 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67763179 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29218653 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936Ðk[ŠÜ'ÜÖO YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184080316300800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚175105508322304 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9217666345472 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 107709057 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚175518440598528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚179707145773524 % _timestamp‚1.7858145386834567e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚175105525361664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9217529379840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8974895408128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚184736106944000 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67763179 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29218653 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 107709057 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚69378450 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚175518415694848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 175481892 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚10199 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 107718713 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8952929235176 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40159795 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448­wúŒË2Ý- _wandbruntime‚4284  _runtime‚4284•!ü9á&ÞÛM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚175122675085312 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67769820 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8953720905274 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚175122675085312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚179724763419066 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8953806708510 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 107719580 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29221258 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29221512 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚170771059553152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚175535622811648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚184098467159552 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚175535647715328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚69385246 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69384990 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67769822 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚4284.221233624 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8975775034880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 175499072 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 107729250 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚184754054821376 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9218564773888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚175535647715328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚69385246 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69384990 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67769822 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚4284.221233624 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8975775034880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 175499072 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 107729250 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚184754054821376 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9218564773888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚175122675085312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚179724763419066 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8953806708510 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 107719580 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29221258 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29221512 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚170771059553152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888®3lqÇ2à- _wandbruntime‚4284  _runtime‚4284ê¹(÷õáï 2loss/SRqqt1/assignment_loss‚0.8415982285019966 2loss/SRqqt2/assignment_loss‚0.7357678169384598 2loss/FBt1/assignment_loss‚0.002869035421401378 /loss/FBt2/detection_loss‚1.0342138598466084 epoch‚0 trainer/global_step‚10199 0loss/FRt1/assignment_loss‚2.6445796627264757 /loss/FRt2/assignment_loss‚2.830815150187566 loss/FBt2/assignment_loss‚0 1loss/SRqqt2/detection_loss‚1.2337670793446402 &loss/total_loss‚16.597693017870817 % _timestamp‚1.7858145390321696e+09 .loss/FRt2/detection_loss‚2.488262836094741 .loss/FBt1/detection_loss‚1.162772715681543 _runtime‚4284.221971087 _step‚815 /loss/FRt1/detection_loss‚2.4602216383913063 1loss/SRqqt1/detection_loss‚1.1628249947360791ƒ~Sàúâô 2loss/SRqqt1/assignment_loss‚0.8415982285019966 2loss/SRqqt2/assignment_loss‚0.7357678169384598 2loss/FBt1/assignment_loss‚0.002869035421401378 /loss/FBt2/detection_loss‚1.0342138598466084 epoch‚0 trainer/global_step‚10199 0loss/FRt1/assignment_loss‚2.6445796627264757 /loss/FRt2/assignment_loss‚2.830815150187566 loss/FBt2/assignment_loss‚0 1loss/SRqqt2/detection_loss‚1.2337670793446402 &loss/total_loss‚16.597693017870817 % _timestamp‚1.7858145390321696e+09 .loss/FRt2/detection_loss‚2.488262836094741 .loss/FBt1/detection_loss‚1.162772715681543 _runtime‚4284.221971087 _step‚815 /loss/FRt1/detection_loss‚2.4602216383913063 1loss/SRqqt1/detection_loss‚1.1628249947360791¯Ä]œã‚(:‘ ¼ÅÓ‹„¡ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚123.05' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚41.016666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Á<,qä‚(:æ  ¼ÅÓ£úȃ disk.dm-0.in‚ 440.05859375 disk.drbd1356.out‚0 disk.drbd1584.in‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 network.sent‚ 9.49514e+06# disk./.usageGB‚589.348258972168 disk.drbd2111.in‚0 disk.sda1.in‚0 disk.drbd1938.out‚0 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.drbd1828.in‚0 disk.drbd1673.in‚0 disk.rbd1.out‚ 14.81640625 disk.sda.out‚ 179.03515625 disk.drbd1550.out‚0 disk.sda1.out‚0) proc.memory.percent‚17.17219352722168 disk.drbd2118.out‚0% disk.nvme0n1.out‚3949.11474609375 disk.drbd2041.in‚0 disk.drbd1285.out‚0% memory_percent‚41.246312856674194 disk.drbd1965.in‚0 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.drbd1817.out‚0 disk.drbd1828.out‚0" disk.dm-0.out‚3949.11474609375 disk.drbd2111.out‚0 disk.rbd0.out‚0) disk./.usagePercent‚32.95943985129021 disk.drbd2118.in‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1965.out‚0 proc.cpu.threads‚29 disk.drbd1938.in‚0 disk.drbd1931.out‚0 disk.drbd1550.in‚0 disk.drbd1552.in‚0 disk.sda2.out‚ 179.03515625 disk.drbd2078.in‚0 disk.drbd1584.out‚0 disk.rbd1.in‚0 disk.drbd1289.out‚0! proc.memory.rssMB‚ 11253.96875 network.recv‚612377 disk.sda2.in‚ 11.6328125 disk.rbd0.in‚0 disk.drbd1510.out‚0 disk.drbd2041.out‚0* proc.memory.availableMB‚38504.81640625 cpu‚13.737339324648081 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd1817.in‚0 disk.drbd1356.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1226.out‚0&xã2å- _wandbruntime‚4304  _runtime‚4304=v.Èæ ' lr-AdamW/pg1‚0.00026413913928790216 ' lr-AdamW/pg2‚0.00026413913928790216 trainer/global_step‚10249 $ _timestamp‚1.785814559104356e+09 _runtime‚ 4304.29430555 _step‚816ÒÓlbÍçÇ ' lr-AdamW/pg1‚0.00026413913928790216 ' lr-AdamW/pg2‚0.00026413913928790216 trainer/global_step‚10249 $ _timestamp‚1.785814559104356e+09 _runtime‚ 4304.29430555 _step‚816°¤ŒW2è- _wandbruntime‚4304  _runtime‚4304¼ô×'éÑO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40356645 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚171591327996832 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚175963846472704 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 trainer/global_step‚10249 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚68095327 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚69719573 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚176378796438528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚176378771534848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 108246715 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9018804113920 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68095327 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚68095329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9018804113920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40356647 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚175963846472704 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚180588145695516 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 108237059 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29362670 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68095329 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29362926 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9262700231680 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚184982755055616 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚185641471766528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9018891543552 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 176342044 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚185641629441536 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9262833003008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚256 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚69719315 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚180588042853032 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8996817698684 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚184982650586624 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚171591310957472 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 108246715 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 % _timestamp‚1.7858145591053011e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184982650586624 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 J#„× êÖO UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚184982755055616 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚185641471766528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9018891543552 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 176342044 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚185641629441536 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9262833003008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚258 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚256 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚69719315 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚180588042853032 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8996817698684 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚184982650586624 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚171591310957472 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 108246715 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 % _timestamp‚1.7858145591053011e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184982650586624 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚176378771534848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 108246715 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9018804113920 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68095327 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚68095329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9018804113920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40356647 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚175963846472704 YKDeviceStatsMonitor.on_train_bat^¼ch_start/reserved_bytes.small_pool.allocated‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚180588145695516 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 108237059 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29362670 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68095329 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29362926 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9262700231680 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚10249 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚180605660498574 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69726096 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚176396003555328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68101972 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚175981030275072 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 176349552 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚175981013235712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 140890624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 176359224 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9263613543936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9019771170304 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚185659733086208 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 176359224 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8997695172018 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68101970 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 108257252 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚171608051129792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9263729530880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚171608068169152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8997609368782 _step‚818 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚185000801445376 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68101970 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚180605763341170 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9019683733504 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68101972 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 —7=å&íßM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚185000801445376 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68101970 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚180605763341170 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9019683733504 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68101972 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 108247582 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚234 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚176395978651648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 trainer/global_step‚10249 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚180605660498574 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69726096 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚176396003555328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68101972 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚175981030275072 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 176349552 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚175981013235712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 140890624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 176359224 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9263613543936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9019771170304 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚185659733086208 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 176359224 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8997695172018 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68101970 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 108257252 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚171608051129792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9263729530880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚171608068169152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8997609368782 _step‚818²Lbe2î- _wandbruntime‚4304  _runtime‚4304@K•±ï % _timestamp‚1.7858145594490488e+09 /loss/FRt1/assignment_loss‚2.363993087235619 /loss/FRt2/assignment_loss‚2.484372341297042 2loss/SRqqt2/assignment_loss‚0.6697419741057924 4loss/FBt2/assignment_loss‚1.9549577565932714e-05 1loss/SRqqt2/detection_loss‚1.0069385476410389 /loss/FBt2/detection_loss‚0.8175390668006407 %loss/total_loss‚16.21488490612248 trainer/global_step‚10249 .loss/FRt1/detection_loss‚2.350011992279221 _runtime‚4304.638843505 _step‚819 2loss/FBt1/assignment_loss‚0.023299289801442455 .loss/FRt2/detection_loss‚2.752157936843348 /loss/FBt1/detection_loss‚1.6531824929080903 2loss/SRqqt1/assignment_loss‚0.9205142242055193 0loss/SRqqt1/detection_loss‚1.173114403427161 epoch‚0ô­¨ ð† 2loss/FBt1/assignment_loss‚0.023299289801442455 .loss/FRt2/detection_loss‚2.752157936843348 /loss/FBt1/detection_loss‚1.6531824929080903 2loss/SRqqt1/assignment_loss‚0.9205142242055193 0loss/SRqqt1/detection_loss‚1.173114403427161 epoch‚0 % _timestamp‚1.7858145594490488e+09 /loss/FRt1/assignment_loss‚2.363993087235619 /loss/FRt2/assignment_loss‚2.484372341297042 2loss/SRqqt2/assignment_loss‚0.6697419741057924 4loss/FBt2/assignment_loss‚1.9549577565932714e-05 1loss/SRqqt2/detection_loss‚1.0069385476410389 /loss/FBt2/detection_loss‚0.8175390668006407 %loss/total_loss‚16.21488490612248 trainer/global_step‚10249 .loss/FRt1/detection_loss‚2.350011992279221 _runtime‚4304.638843505 _step‚819³áœe°œñ‚(:‘ Ÿ¼ÅÓÓ¬§ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚129.39' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.129999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0L¡¦Ýrò‚(:ç  Ÿ¼ÅÓéäÚƒ disk.dm-0.in‚ 440.05859375 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd1510.in‚0 network.sent‚ 9.540111e+06 disk.drbd1817.in‚0 disk.drbd1965.out‚0 disk.rbd0.in‚0* proc.memory.availableMB‚38504.04296875 network.recv‚613845 disk.drbd1938.in‚0 disk.sda.in‚ 11.6328125 disk.drbd2111.out‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.drbd1226.out‚0 disk.drbd2078.in‚0 disk.drbd2111.in‚0 disk.drbd2118.in‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1817.out‚0 disk.rbd1.out‚14.87939453125) disk./.usagePercent‚32.95943985129021 disk.drbd1510.out‚0 disk.drbd1285.out‚0) proc.memory.percent‚17.17219352722168 disk.drbd1289.out‚0 disk.sda.out‚ 179.50390625 disk.drbd1828.in‚0 disk.sda2.out‚ 179.50390625# disk./.usageGB‚589.348258972168 disk.drbd1552.in‚0$ disk.nvme0n1.out‚3949.3349609375 disk.drbd1356.in‚0$ memory_percent‚41.24749302864075 disk.drbd1673.in‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.drbd1584.in‚0 proc.cpu.threads‚29 disk.sda1.in‚0 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1285.in‚0 disk.drbd1828.out‚0 disk.sda2.in‚ 11.6328125! disk.dm-0.out‚3949.3349609375 disk.drbd1289.in‚0 disk.drbd2118.out‚0 disk.drbd1226.in‚0! proc.memory.rssMB‚ 11253.96875 cpu‚13.737353273187974 disk.drbd1938.out‚0 disk.drbd1356.out‚0ûþZ¥ó‚(:’ ®¼ÅÓÐÑ„ƒ gpu.0.gpu‚21.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚111.115' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.038333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0òÑjµnô‚(:ã  ®¼ÅÓîõȃ disk.drbd1938.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1828.out‚0 disk.sda1.out‚0 disk.drbd1817.out‚0$ disk.nvme0n1.out‚3949.6044921875 disk.drbd1931.in‚0 disk.drbd1931.out‚0 cpu‚13.737112271711409 disk.drbd1550.in‚0 disk.dm-0.in‚ 440.05859375 disk.drbd2118.out‚0 disk.drbd2078.out‚0 disk.rbd0.out‚0 disk.drbd2118.in‚0 disk.drbd1938.out‚0 disk.rbd1.out‚14.87939453125 proc.cpu.threads‚29 disk.drbd1673.in‚0 disk.drbd1584.in‚0 disk.drbd1226.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1673.out‚0 disk.drbd1285.in‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0 disk.drbd1552.out‚0* disk./.usagePercent‚32.959442411344305 disk.sda1.in‚0 disk.drbd1817.in‚0* proc.memory.availableMB‚38504.53515625$ disk./.usageGB‚589.3483047485352 disk.drbd2041.out‚0 disk.drbd2078.in‚0 disk.sda2.out‚ 179.71875$ memory_percent‚41.24674201011658 disk.drbd2041.in‚0! disk.dm-0.out‚3949.6044921875) proc.memory.percent‚17.17219352722168 network.recv‚615571 disk.drbd1965.in‚0 disk.drbd1965.out‚0! proc.memory.rssMB‚ 11253.96875 disk.drbd1226.out‚0 disk.drbd2111.out‚0 disk.sda.out‚ 179.71875 disk.drbd1289.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚ 440.05859375 network.sent‚ 9.585199e+06 disk.drbd1584.out‚0 disk.drbd2111.in‚0 disk.drbd1552.in‚0 disk.drbd1356.out‚0 disk.drbd1550.out‚0 disk.drbd1510.in‚0 disk.sda2.in‚ 11.6328125 disk.drbd1356.in‚0 disk.rbd0.in‚0tÌ6¿2õ- _wandbruntime‚4325  _runtime‚4325—.@6ÊöÄ ' lr-AdamW/pg1‚0.00026542774861216753 ' lr-AdamW/pg2‚0.00026542774861216753 trainer/global_step‚10299 % _timestamp‚1.7858145801555738e+09 _runtime‚4325.347619624 _step‚820ËX@Ï÷É ' lr-AdamW/pg1‚0.00026542774861216753 ' lr-AdamW/pg2‚0.00026542774861216753 trainer/global_step‚10299 % _timestamp‚1.7858145801555738e+09 _runtime‚4325.347619624 _step‚820´ö8zÁ2ø- _wandbruntime‚4325  _runtime‚4325'eÏ19ùÐO M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 108765061 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚185885089341440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 177202196 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29505642 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚10299 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚185884984872448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 170257920 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29505924 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 4325.34770381 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 108774717 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9062887678976 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_L_Ôpool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9062800249344 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚176822184623104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68427479 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚282 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40553495 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _step‚821 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚284 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚177239127374848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9040706162192 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9062887678976 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚172428336612832 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68427477 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9040620359068 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9307880433664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚181468939932540 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚176822201662464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚185884984872448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9308025787904DdÛ'úÕO ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9062800249344 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚70059137 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68427477 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9040620359068 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9307880433664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚181468939932540 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚176822201662464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚185884984872448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9308025787904 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 108765061 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚185885089341440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 177202196 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29505642 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚10299 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚185884984872448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 170257920 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29505924 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 4325.34770381 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 108774717 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9062887678976 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9062800249344 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚176822184623104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68427479 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚282 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40553495 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 _step‚821 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚284 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚177239127374848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9040706162192 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9062887678976 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚172428336612832 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 177219376 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚172445076785152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68434120 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚176839368425472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29508642 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9063679868928 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚70066336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9308918352896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚822 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 177209704 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70066074 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68434122 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9063767305728 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68434120 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 108785254 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚181486557578082 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚185903135731200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚172445059745792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 177219376 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0›gSýßM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚176839368425472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚177256359395328 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚176839368425472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29508642 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9063679868928 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚70066336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9308918352896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚822 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 177209704 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70066074 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68434122 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9063767305728 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68434120 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 108785254 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚181486557578082 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚185903135731200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚172445059745792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 177219376 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9041583635526 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 108785254 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40557434 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 177219376 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚172445076785152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68434120 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EŽ4Ó €‡ 2loss/SRqqt2/assignment_loss‚0.8637557713744732 .loss/FRt1/detection_loss‚2.307020775573228 1loss/SRqqt1/detection_loss‚1.0534685813145686 $ _timestamp‚1.785814580503249e+09 _step‚823 0loss/FRt1/assignment_loss‚2.5896733422433176 0loss/SRqqt2/detection_loss‚1.208827443420887 .loss/FBt1/detection_loss‚1.205039533889956 _runtime‚4325.693021143 /loss/FRt2/assignment_loss‚2.601625551412135 1loss/SRqqt1/assignment_loss‚1.039983460414925 4loss/FBt1/assignment_loss‚2.0331557664121647e-06 /loss/FBt2/detection_loss‚1.1638718798587269 trainer/global_step‚10299 3loss/FBt2/assignment_loss‚6.068804933337643e-07 /loss/FRt2/detection_loss‚2.4994443125194974 &loss/total_loss‚16.532713292057974 epoch‚0·D¿çmœ‚(:‘ ½¼ÅÓ„‰ªƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚104.141' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚34.71366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0çNi‚‚(:Þ  ½¼ÅÓ–·Öƒ disk.drbd1584.out‚0 disk.drbd2041.in‚0 disk.drbd1550.in‚0 disk.sda2.out‚ 179.953125 disk.drbd1931.in‚0 disk.drbd2078.in‚0 disk.dm-0.out‚ 3949.8984375 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.rbd1.in‚0 disk.drbd1289.out‚0* proc.memory.availableMB‚38504.72265625! proc.memory.rssMB‚ 11253.96875 network.recv‚616468 disk.sda2.in‚ 11.6328125 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1356.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1965.in‚0 disk.sda1.in‚0% memory_percent‚41.246455907821655 network.sent‚ 9.588243e+06 disk.drbd1931.out‚0 disk.drbd2041.out‚0 disk.drbd1285.out‚0 cpu‚13.736956596436963 disk.rbd1.out‚ 14.9423828125 disk.drbd1226.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚ 440.05859375 disk.drbd1673.in‚0 disk.drbd1226.out‚0 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.sda.out‚ 179.953125 disk.drbd1965.out‚0 disk.drbd1938.out‚0 disk.drbd1550.out‚0 disk.drbd1510.out‚0 disk.drbd1552.out‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1584.in‚0 disk.rbd0.out‚0) disk./.usagePercent‚32.95944582474976 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 proc.cpu.threads‚29$ disk./.usageGB‚589.3483657836914 disk.drbd1938.in‚0 disk.drbd1356.out‚0 disk.drbd2118.out‚0! disk.nvme0n1.out‚ 3949.8984375 disk.drbd1285.in‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0) proc.memory.percent‚17.17219352722168I¬l±2ƒ- _wandbruntime‚4345  _runtime‚4345þ©È„ _step‚824 & lr-AdamW/pg1‚0.0002667163579364328 & lr-AdamW/pg2‚0.0002667163579364328 trainer/global_step‚10349 % _timestamp‚1.7858146007108793e+09 _runtime‚4345.900824323ÔCˆ‹Í…Ç % _timestamp‚1.7858146007108793e+09 _runtime‚4345.900824323 _step‚824 & lr-AdamW/pg1‚0.0002667163579364328 & lr-AdamW/pg2‚0.0002667163579364328 trainer/global_step‚10349¸Õ5Çn2†- _wandbruntime‚4345  _runtime‚43457ªÛ×'‡ÑO QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 109302719 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚173265345228832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚173265328189472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚177680522773504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29649043 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9106883814400 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚186787319158272 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68759629 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚177680522773504 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚187452697537024 _step‚825 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚178099508118528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9084594625700 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9084508822576 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚186787319158272 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9353189418496 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚182349837012048 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40750347 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚186787423627264 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 178052690 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚68759627 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 109293063 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9106883814400 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9353056647168 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚182349939854532 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 109302719 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40750345 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9106796384768 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121XÔÜ'ˆÖO JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9106883814400 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚186787319158272 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68759629 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚177680522773504 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚187452697537024 _step‚825 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚244 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚178099508118528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9084594625700 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9084508822576 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚186787319158272 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9353189418496 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚182349837012048 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40750347 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚186787423627264 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 178052690 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚68759627 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 109293063 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9106883814400 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9353056647168 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚182349939854532 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 109302719 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40750345 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9106796384768 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 109302719 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚173265345228832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚246 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚173265328189472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚177680522773504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29649043¹@…êã2‰- _wandbruntime‚4346  _runtime‚4346ßÞŸLâŠÛM SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 109303586 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68766272 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9354093786112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 178079528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚187470809021440 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 178069856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9107676004352 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚178116715235328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9085472099034 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚177697706575872 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 109303586 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚173282085401152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚186805470017024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚177697689536512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 178069856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68766270 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 178079528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29651903 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29651648 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9107763441152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 EDeviceS"‘Å¥ÿtatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚186805470017024 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚177697689536512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9353956827648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40754284 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70405930 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚173282068361792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9107763441152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 % _timestamp‚1.7858146010587747e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚826 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 trainer/global_step‚10349 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 109313256 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9085386295798¹<õÊæ&‹àM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚177697689536512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9353956827648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40754284 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70405930 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚173282068361792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9107763441152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 % _timestamp‚1.7858146010587747e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚826 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 trainer/global_step‚10349 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 109313256 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9085386295798 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 109303586 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68766272 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9354093786112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 178079528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚187470809021440 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 178069856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9107676004352 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚178116715235328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9085472099034 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚177697706575872 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 109303586 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚173282085401152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚186805470017024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚177697689536512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 178069856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68766270 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 178079528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29651903 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29651648 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9107763441152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚186805470017024 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 178922500 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚188358235782144 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 109821065 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9150792520192 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40947197 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29792393 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚187689757913088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9150879949824 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69091777 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚270 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 109830721 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚187689653444096 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚178538877963264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69091777 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚178538860923904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9150879949824 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚174102336805472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚188358073912832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚10399 _step‚829 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9128483089208 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _runtime‚4366.671644002 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40947195 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚187689653444096 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 109830721 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69091779 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚268 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚178959839054848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9398371823616 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚183230836934040 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚183230734091556 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 % _timestamp‚1.7858146214780278e+09 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69091779 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚178538877963264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9398234857984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚70739858 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚178959863958528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚174102353844832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 178912842 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚187689757913088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 109821065 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚178538860923904 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚188358073912832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚10399 _step‚829 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9128483089208 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 _runtime‚4366.671644002 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40947195 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚187689653444096 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 109830721 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69091779 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚268 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚178959839054848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9398371823616 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚183230836934040 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚183230734091556 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 % _timestamp‚1.7858146214780278e+09 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69091779 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚178538877963264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9398234857984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚70739858 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚178959863958528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚174102353844832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 178912842 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚187689757913088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 109821065 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚178538860923904 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 161869312 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 178922500 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚188358235782144 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 109821065 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9150792520192 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40947197 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29792393½1Ç¿2™- _wandbruntime‚4367  _runtime‚4367)§Õà&šÚM REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40951132 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 178939680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 178930008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚178556044726272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29795229 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9129274759306 trainer/global_step‚10399 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚70746607 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 109831588 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69098420 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9151672139776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚178556044726272 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚178977071075328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70746361 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚183248351737098 _step‚830 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9151759576576 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 178930008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9151672139776 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 178939680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚183248454579694 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚174119094017152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69098422 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚188376201929728 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 4367.01780077 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9151759576576 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69098422 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚178556027686912 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 109841258 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29795473 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚187707699826688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚178977046171648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 % _timestamp‚1.7858146218275034e+09;£_*å&›ßM L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 178939680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚246 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚183248454579694 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚174119094017152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69098422 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 149279232 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚188376201929728 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 4367.01780077 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9151759576576 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69098422 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚178556027686912 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 109841258 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29795473 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚187707699826688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚178977046171648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 % _timestamp‚1.7858146218275034e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40951132 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 178939680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 178930008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚178556044726272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29795229 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9129274759306 trainer/global_step‚10399 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚70746607 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 109831588 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69098420 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9151672139776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚178556044726272 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚178977071075328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70746361 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚183248351737098 _step‚830 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9151759576576 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 178930008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9151672139776 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0¾Ž¨l`2œ- _wandbruntime‚4367  _runtime‚4367ÓR¿V‚ /loss/FRt1/assignment_loss‚2.552423291546958 /loss/FRt2/assignment_loss‚2.887997662409758 3loss/FBt2/assignment_loss‚1.882251753290563e-08 /loss/FBt1/detection_loss‚0.9277245767135173 epoch‚0 trainer/global_step‚10399 % _timestamp‚1.7858146218286927e+09 2loss/SRqqt2/assignment_loss‚0.7774240047814182 3loss/FBt1/assignment_loss‚6.367191752872259e-05 /loss/FBt2/detection_loss‚1.2291661835839285 %loss/total_loss‚16.58596993302941 _runtime‚4367.018457862 _step‚831 /loss/FRt2/detection_loss‚2.5247464390137258 2loss/SRqqt1/assignment_loss‚0.7645129021830284 -loss/FRt1/detection_loss‚2.59984962535756 1loss/SRqqt1/detection_loss‚1.0585244378218284 1loss/SRqqt2/detection_loss‚1.2635371188776425o&Ç ž‡ 2loss/SRqqt1/assignment_loss‚0.7645129021830284 -loss/FRt1/detection_loss‚2.59984962535756 1loss/SRqqt1/detection_loss‚1.0585244378218284 1loss/SRqqt2/detection_loss‚1.2635371188776425 /loss/FRt1/assignment_loss‚2.552423291546958 /loss/FRt2/assignment_loss‚2.887997662409758 3loss/FBt2/assignment_loss‚1.882251753290563e-08 /loss/FBt1/detection_loss‚0.9277245767135173 epoch‚0 trainer/global_step‚10399 % _timestamp‚1.7858146218286927e+09 2loss/SRqqt2/assignment_loss‚0.7774240047814182 3loss/FBt1/assignment_loss‚6.367191752872259e-05 /loss/FBt2/detection_loss‚1.2291661835839285 %loss/total_loss‚16.58596993302941 _runtime‚4367.018457862 _step‚831 /loss/FRt2/detection_loss‚2.5247464390137258¿øÉ/ÆœŸ‚(:‘ ê¼ÅÓ¾ˆ¨ƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚125.459' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.81966666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0<˜! p ‚(:å  ê¼ÅÓƒ™çƒ disk.drbd2078.out‚0 disk.drbd1289.out‚0 disk.sda1.out‚0 disk.drbd1965.in‚0 disk.drbd2118.in‚0 disk.drbd1226.in‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.drbd1673.out‚0 disk.drbd1356.in‚0 disk.drbd1550.out‚0 disk.drbd1938.in‚0( proc.memory.availableMB‚ 38504.546875 proc.cpu.threads‚29 disk.drbd2118.out‚0 disk.sda2.out‚ 181.0859375 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0) disk./.usagePercent‚32.95956678730552# disk./.usageGB‚589.350528717041 disk.drbd1550.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚ 440.05859375 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 11.6328125" disk.dm-0.out‚3950.82568359375 disk.drbd1965.out‚0 disk.drbd1931.in‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd1828.out‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.sda.in‚ 11.6328125% disk.nvme0n1.out‚3950.82568359375! proc.memory.rssMB‚ 11253.96875 cpu‚13.736328226733042 network.sent‚ 9.681345e+06 disk.drbd2041.in‚0 disk.rbd1.out‚ 14.9423828125 disk.drbd1226.out‚0 disk.nvme0n1.in‚ 440.05859375 disk.drbd1584.out‚0% memory_percent‚41.246724128723145 network.recv‚620466 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.drbd1552.in‚0 disk.rbd1.in‚0) proc.memory.percent‚17.17219352722168 disk.sda.out‚ 181.0859375 disk.drbd2111.in‚0ôÝ!2¡- _wandbruntime‚4387  _runtime‚4387 ;8Ȣ & lr-AdamW/pg1‚0.0002692935765849634 & lr-AdamW/pg2‚0.0002692935765849634 trainer/global_step‚10449 % _timestamp‚1.7858146424286218e+09 _runtime‚4387.618564801 _step‚832ø½õqꣂ & lr-AdamW/pg1‚0.0002692935765849634 & lr-AdamW/pg2‚0.0002692935765849634 trainer/global_step‚10449 % _timestamp‚1.7858146424286218e+09 _runtime‚4387.618564801 _step‚832Àh†vþ2¤- _wandbruntime‚4387  _runtime‚4387£P¤õ×'¥ÑO GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚833 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858146424296055e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 179782652 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9172371552716 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚188592092198912 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚179397199074304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9194788655616 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 110349067 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41144045 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚188591987729920 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9443486605312 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚188592092198912 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚174939345421472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9172285749592 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚188591987729920 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚184111734013548 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 110358723 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69423927 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69423927 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚179820194894848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9194876085248 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚71080134 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚4387.623645128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚174939362460832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29936087 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9194788655616 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 179772994 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29935835 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69423929 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚10449 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69423929 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚189263557117440oÅýÜ'¦ÖO QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41144045 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚188591987729920 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9443486605312 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚188592092198912 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚174939345421472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9172285749592 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚188591987729920 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚184111734013548 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 110358723 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69423927 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69423927 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚179820194894848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9194876085248 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚71080134 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚4387.623645128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚174939362460832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29936087 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9194788655616 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 179772994 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29935835 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69423929 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 trainer/global_step‚10449 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69423929 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚189263557117440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚833 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7858146424296055e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 179782652 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9172371552716 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚188592092198912 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚179397199074304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9194788655616 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 110349067 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0Áñï´2§- _wandbruntime‚4387  _runtime‚4387³hÖ`ß&¨ÙM TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚179414382876672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚189281822295552 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚189281668822016 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚174956102633152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 179790160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚179414365837312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9444395380224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9195755712000 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚184129351659202 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9444266810368 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 110359590 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9195668275200 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚71086885 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71086637 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69430572 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69430570 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚174956085593792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚ 4387.96957278 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9195755712000 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 179799832 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 110359590 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚179837402011648 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29938901 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 $ _timestamp‚1.785814642779274e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚188610138588672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41147982 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚179837426915328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9173163222814 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 110369260 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41147984 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121äÉW ©ÞM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚179414365837312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚248 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9444395380224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9195755712000 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚184129351659202 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9444266810368 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 110359590 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9195668275200 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚71086885 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71086637 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69430572 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69430570 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚174956085593792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _runtime‚ 4387.96957278 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9195755712000 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 179799832 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 110359590 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚179837402011648 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29938901 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 $ _timestamp‚1.785814642779274e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚188610138588672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41147982 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚179837426915328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9173163222814 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 110369260 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41147984 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚179414382876672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚189281822295552 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚189281668822016 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚174956102633152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚246 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 179790160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130ž£Ê2ª- _wandbruntime‚4387  _runtime‚4387`ª5Íõ«ï 4loss/FBt1/assignment_loss‚3.8264592319253986e-05 % _timestamp‚1.7858146427804627e+09 1loss/SRqqt2/assignment_loss‚0.912002529295634 loss/FBt2/assignment_loss‚0 1loss/SRqqt1/detection_loss‚1.1284043667623056 /loss/FBt1/detection_loss‚0.8502644390932151 _step‚835 /loss/FRt1/assignment_loss‚2.226693415488952 0loss/FRt2/assignment_loss‚2.6798851999004234 1loss/SRqqt1/assignment_loss‚0.873931867765406 .loss/FRt1/detection_loss‚2.525116389569564 /loss/FRt2/detection_loss‚2.4699086443092044 1loss/SRqqt2/detection_loss‚1.2538539261571062 trainer/global_step‚10449 /loss/FBt2/detection_loss‚0.8662500855280086 %loss/total_loss‚15.78634912846214 epoch‚0 _runtime‚4387.9702442622û{Lú¬ô 4loss/FBt1/assignment_loss‚3.8264592319253986e-05 % _timestamp‚1.7858146427804627e+09 1loss/SRqqt2/assignment_loss‚0.912002529295634 loss/FBt2/assignment_loss‚0 1loss/SRqqt1/detection_loss‚1.1284043667623056 /loss/FBt1/detection_loss‚0.8502644390932151 _step‚835 /loss/FRt1/assignment_loss‚2.226693415488952 0loss/FRt2/assignment_loss‚2.6798851999004234 1loss/SRqqt1/assignment_loss‚0.873931867765406 .loss/FRt1/detection_loss‚2.525116389569564 /loss/FRt2/detection_loss‚2.4699086443092044 1loss/SRqqt2/detection_loss‚1.2538539261571062 trainer/global_step‚10449 /loss/FBt2/detection_loss‚0.8662500855280086 %loss/total_loss‚15.78634912846214 epoch‚0 _runtime‚4387.970244262Ãfà­‚(:’ ù¼ÅÓ“Ƨƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚132.885' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.294999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0#)Íyn®‚(:ã  ù¼ÅÓò§ºƒ disk.drbd1510.in‚0 disk.drbd1510.out‚0# disk.nvme0n1.out‚3951.318359375 disk.drbd1584.in‚0 disk.drbd1965.out‚0 cpu‚13.736208268680409 disk.dm-0.out‚3951.318359375 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 11.6328125 disk.drbd1552.in‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1550.out‚0 disk.drbd1226.out‚0 disk.drbd1356.in‚0 proc.cpu.threads‚29 network.sent‚ 9.726357e+06 disk.drbd2111.in‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.drbd2078.out‚0 disk.drbd1289.in‚0 disk.sda1.out‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.drbd2041.out‚0$ disk./.usageGB‚589.3504905700684 disk.sda1.in‚0 disk.drbd1226.in‚0 disk.drbd1817.in‚0 disk.nvme0n1.in‚ 440.05859375% memory_percent‚41.246843338012695 disk.drbd1965.in‚0 disk.drbd1550.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.sda2.out‚ 181.39453125 disk.drbd1356.out‚0) disk./.usagePercent‚32.95956465392711 disk.drbd1938.out‚0 disk.drbd2078.in‚0 network.recv‚622114 disk.rbd0.out‚0' proc.memory.availableMB‚ 38504.46875) proc.memory.percent‚17.17219352722168 disk.drbd1285.out‚0 disk.sda.out‚ 181.39453125! proc.memory.rssMB‚ 11253.96875 disk.drbd1285.in‚0 disk.rbd1.out‚ 14.9423828125 disk.rbd0.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.dm-0.in‚ 440.05859375 disk.drbd2111.out‚0 disk.drbd1584.out‚0 Z•t2¯- _wandbruntime‚4408  _runtime‚4408€çó—È°Â & lr-AdamW/pg2‚0.0002705821859092287 trainer/global_step‚10499 % _timestamp‚1.7858146630231473e+09 _runtime‚4408.213071833 _step‚836 & lr-AdamW/pg1‚0.0002705821859092287äoQ¥Í±Ç % _timestamp‚1.7858146630231473e+09 _runtime‚4408.213071833 _step‚836 & lr-AdamW/pg1‚0.0002705821859092287 & lr-AdamW/pg2‚0.0002705821859092287 trainer/global_step‚10499Äörf’2²- _wandbruntime‚4408  _runtime‚4408µ5 à×'³ÑO UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚190169075411456 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 110886725 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9238784791040 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9488524676608 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69756077 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69756077 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚180680575638528 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚180255554264064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9238784791040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41340895 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚189494322015744 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚249 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30079590 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30079343 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 180642804 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9216174213100 % _timestamp‚1.7858146630240371e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 110877069 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚4408.217637258 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚184992528250572 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9216260016224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚175776354037472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9238872220672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚180255554264064 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69756079 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 110877069 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚10499 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚180255537224704 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚190169233086464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚175776371076832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69756079 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚180255537224704 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚71420238¼ì]™Ü'´ÖO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚190169075411456 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 110886725 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9238784791040 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9488524676608 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69756077 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69756077 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚180680575638528 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚180255554264064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9238784791040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41340895 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚189494322015744 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚249 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30079590 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30079343 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 180642804 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9216174213100 % _timestamp‚1.7858146630240371e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 110877069 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 _runtime‚4408.217637258 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚184992528250572 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9216260016224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚175776354037472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9238872220672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚180255554264064 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69756079 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 110877069 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 trainer/global_step‚10499 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚180255537224704 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚190169233086464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚175776371076832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69756079 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚180255537224704 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚71420238 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712Å ³ ˆ2µ- _wandbruntime‚4408  _runtime‚4408C€Ž3¼¶ÛM SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=Device=ÉžÒ%&StatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30082514 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9489442314240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚185010145896114 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚190187359930880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9489577175552 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 180650312 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185010248738710 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30082254 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71427086 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9217137489558 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚180272703987712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚180272721027072 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚180697782755328 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 110887592 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69762722 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9239751847424 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41344832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858146633721185e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9239751847424 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚4408.562392957 _step‚838 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚180272703987712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚189512368398336 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 180659984 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 110887592 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69762720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚175793111249152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9217051686322 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚10499 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9239664410624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 180650312 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69762720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚175793111249152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9217051686322 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 trainer/global_step‚10499 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9239664410624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 180650312 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 180650312 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185010248738710 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30082254 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71427086 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9217137489558 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚180272703987712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚180272721027072 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚180697782755328 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 110887592 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69762722 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9239751847424 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41344832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚260 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7858146633721185e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9239751847424 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚4408.562392957 _step‚838 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚180272703987712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚189512368398336 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 180659984 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 110887592 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚181113875375104 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9260148479732 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9260062676608 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 111405071 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9282780926464 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9533892076544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185873528172564 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70088229 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 181502956 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9282868356096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70088227 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚190396656301568 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9533757208064 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70088227 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30223773 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 111414727 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚181113892414464üÛ'ÄÕO WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚190396656301568 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚181113875375104 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚191074823555072 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30224021 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚176613362653472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚181113875375104 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9260148479732 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9260062676608 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 111405071 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9282780926464 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9533892076544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185873528172564 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70088229 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 181502956 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9282868356096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70088227 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚190396656301568 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9533757208064 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70088227 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚250 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30223773 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 111414727 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚181113892414464 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9282780926464 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚71761518 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚185873425330080ÉÓÂC12Å- _wandbruntime‚4429  _runtime‚4429"$¸èá&ÆÛM H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 181510464 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚4429.329106239 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚842 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70094872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚190414807160320 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9283660546048 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 181510464 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70094870 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 111415594 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚190414807160320 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70094870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9283660546048 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚191092777109504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚176630119865152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9261025953066 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚181131042138112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41541684 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9534794084864 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚181558138595328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30226879 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚191092932680192 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9260940149830 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 111425264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9283747982848 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚268 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70094872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚176630102825792 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 181520136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚71768563 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71768293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185891145818218 trainer/global_step‚10549 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 181520136¢ƒzœŽ#ÇàM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚842 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70094872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚190414807160320 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9283660546048 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 181510464 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70094870 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 111415594 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚190414807160320 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70094870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9283660546048 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚191092777109504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚176630119865152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9261025953066 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚181131042138112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41541684 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9534794084864 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚181558138595328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30226879 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚191092932680192 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9260940149830 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 111425264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9283747982848 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚268 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70094872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚176630102825792 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 181520136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚71768563 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 130667008 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71768293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185891145818218 trainer/global_step‚10549 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 181520136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 181510464 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚4429.329106239 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200ÊQ6O2È- _wandbruntime‚4429  _runtime‚4429(Ì&˜ÿÉù 2loss/SRqqt2/assignment_loss‚0.8374424873304273 +loss/FRt1/detection_loss‚2.414934463799 1loss/SRqqt1/detection_loss‚1.0774898913795832 .loss/FRt2/detection_loss‚2.251254856262518 /loss/FBt2/detection_loss‚1.6320492225484207  _timestamp‚1.78581468414e+09 _runtime‚4429.329792182 epoch‚0 .loss/FRt2/assignment_loss‚2.07834190842898 3loss/FBt1/assignment_loss‚0.0005172385017641546 2loss/FBt2/assignment_loss‚5.00662785076254e-06 1loss/SRqqt2/detection_loss‚1.2061891578108308 &loss/total_loss‚15.859925040657817 _step‚843 /loss/FRt1/assignment_loss‚2.451617921801174 2loss/SRqqt1/assignment_loss‚0.8702370917279517 /loss/FBt1/detection_loss‚1.0398457944393158 trainer/global_step‚10549R—þŠÊþ _runtime‚4429.329792182 epoch‚0 .loss/FRt2/assignment_loss‚2.07834190842898 3loss/FBt1/assignment_loss‚0.0005172385017641546 2loss/FBt2/assignment_loss‚5.00662785076254e-06 1loss/SRqqt2/detection_loss‚1.2061891578108308 &loss/total_loss‚15.859925040657817 _step‚843 /loss/FRt1/assignment_loss‚2.451617921801174 2loss/SRqqt1/assignment_loss‚0.8702370917279517 /loss/FBt1/detection_loss‚1.0398457944393158 trainer/global_step‚10549 2loss/SRqqt2/assignment_loss‚0.8374424873304273 +loss/FRt1/detection_loss‚2.414934463799 1loss/SRqqt1/detection_loss‚1.0774898913795832 .loss/FRt2/detection_loss‚2.251254856262518 /loss/FBt2/detection_loss‚1.6320492225484207  _timestamp‚1.78581468414e+09ËvóÆhœË‚(:‘ ¦½ÅÓžüƒƒ gpu.0.gpu‚14.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚116.72' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.906666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¸x«eÌ‚(:Ú  ¦½ÅÓҵʃ disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.drbd1285.out‚0! proc.memory.rssMB‚ 11253.96875) disk./.usagePercent‚32.95956358723791* proc.memory.availableMB‚38503.74609375 network.recv‚626429 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.rbd0.out‚0 disk.drbd2111.out‚0$ memory_percent‚41.24794602394104 disk.drbd1965.in‚0) proc.memory.percent‚17.17219352722168 disk.drbd1285.in‚0 disk.drbd1289.out‚0 disk.drbd1828.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚440.0625 disk.drbd1584.in‚0 disk.sda.out‚ 182.30078125 disk.drbd2078.out‚0 proc.cpu.threads‚29 disk.sda2.in‚ 11.6328125 disk.drbd2111.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1226.out‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 network.sent‚ 9.81941e+06 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.sda2.out‚ 182.30078125# disk./.usageGB‚589.350471496582 disk.drbd1938.out‚0 disk.rbd1.out‚ 14.9423828125 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1550.in‚0 disk.drbd2078.in‚0 disk.dm-0.out‚3952.126953125 disk.drbd1938.in‚0 disk.dm-0.in‚440.0625 disk.drbd1550.out‚0 disk.rbd0.in‚0 cpu‚13.73556803598558 disk.sda1.in‚0 disk.drbd1817.in‚0# disk.nvme0n1.out‚3952.126953125 disk.drbd1965.out‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0 disk.drbd1931.in‚0 disk.drbd2041.out‚0 disk.drbd1226.in‚0 disk.drbd1828.out‚0 Ã02Í- _wandbruntime‚4449  _runtime‚44499¢¬+ÈΠ& lr-AdamW/pg1‚0.0002731594045577593 & lr-AdamW/pg2‚0.0002731594045577593 trainer/global_step‚10599 % _timestamp‚1.7858147044983933e+09 _runtime‚4449.688330408 _step‚844,¨ÀmÍÏÇ trainer/global_step‚10599 % _timestamp‚1.7858147044983933e+09 _runtime‚4449.688330408 _step‚844 & lr-AdamW/pg1‚0.0002731594045577593 & lr-AdamW/pg2‚0.0002731594045577593Ìr%0Ö2Ð- _wandbruntime‚4449  _runtime‚4449[Öµ×'ÑÑO NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30368185 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚177450388308832 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚181972230564864 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚191298990587392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 111942729 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚181972213525504 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9303951140116 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9579084780544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚186754322409588 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70420379 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚191980372099072 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70420377 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9578945717760 trainer/global_step‚10599 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858147044993014e+09 _runtime‚4449.692644799 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚181972230564864 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9326777061888€ÁÌÜ'ÒÖO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚186754322409588 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70420379 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚191980372099072 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70420377 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9578945717760 trainer/global_step‚10599 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858147044993014e+09 _runtime‚4449.692644799 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚181972230564864 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9326777061888 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30368185 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚177450388308832 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚181972230564864 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚191298990587392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 111942729 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚181972213525504 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9303951140116 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9579084780544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251͉µ›2Ó- _wandbruntime‚4450  _runtime‚4450ÿ” Ç2ÔÛM aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9579984462336 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 trainer/global_step‚10599 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚182418494435328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 182370616 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70427022 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70427020 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 182380288 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 182380288 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚181989397327872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9327656681472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41738534 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9579849601024 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9304914416574 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚181989380288512 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 111943596 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72109260 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚181989397327872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70427022 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72109512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.s$Þ¯ mall_pool.current‚ 134861312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 111953266 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚191317141446144 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚182418469531648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚186771940055130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41738532 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9327744118272 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9327744118272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 182370616 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 111943596 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚191998319132672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30370728 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚4450.037854419 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚846 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30370978 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚4450.037854419 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _step‚846 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30370978 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 182370616 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70427022 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70427020 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 182380288 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 182380288 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚181989397327872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9327656681472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41738534 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9579849601024 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9304914416574 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚181989380288512 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 111943596 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72109260 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚181989397327872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70427022 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72109512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 111953266 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚191317141446144 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚182418469531648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚186771940055130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41738532 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121Î,£D2Ö- _wandbruntime‚4450  _runtime‚4450Íu£; ׆ 4loss/FBt2/assignment_loss‚1.3857995527407259e-06 /loss/FRt2/detection_loss‚2.4681860119411625 1loss/SRqqt2/detection_loss‚1.0866418919567742 /loss/FBt1/detection_loss‚0.8382117457401294 0loss/FRt2/assignment_loss‚2.5840974062988438 2loss/SRqqt1/assignment_loss‚0.7080684969284556 1loss/SRqqt1/detection_loss‚1.0884780731383648 /loss/FBt2/detection_loss‚0.9319612674686747 epoch‚0 trainer/global_step‚10599 _step‚847 % _timestamp‚1.7858147048487537e+09 _runtime‚ 4450.03854338 /loss/FRt1/assignment_loss‚2.378006230229917 2loss/SRqqt2/assignment_loss‚0.9241903758149094 4loss/FBt1/assignment_loss‚5.2847805981843685e-06 /loss/FRt1/detection_loss‚2.2723640010732673 %loss/total_loss‚15.28021217117065ÜŽpÏØ‹ 0loss/FRt2/assignment_loss‚2.5840974062988438 2loss/SRqqt1/assignment_loss‚0.7080684969284556 1loss/SRqqt1/detection_loss‚1.0884780731383648 /loss/FBt2/detection_loss‚0.9319612674686747 epoch‚0 trainer/global_step‚10599 _step‚847 % _timestamp‚1.7858147048487537e+09 _runtime‚ 4450.03854338 /loss/FRt1/assignment_loss‚2.378006230229917 2loss/SRqqt2/assignment_loss‚0.9241903758149094 4loss/FBt1/assignment_loss‚5.2847805981843685e-06 /loss/FRt1/detection_loss‚2.2723640010732673 %loss/total_loss‚15.28021217117065 4loss/FBt2/assignment_loss‚1.3857995527407259e-06 /loss/FRt2/detection_loss‚2.4681860119411625 1loss/SRqqt2/detection_loss‚1.0866418919567742 /loss/FBt1/detection_loss‚0.8382117457401294Ï22»;Ù‚(:„ µ½ÅÓ×ܦƒ gpu.0.gpu‚39.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚101.82' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚33.94 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0&rñeÚ‚(:Ú  µ½ÅÓ§ÊÛƒ disk.drbd1965.out‚0 disk.drbd1828.out‚0 disk.sda1.out‚0 disk.drbd1931.in‚0 disk.drbd2078.out‚0' proc.memory.availableMB‚ 38503.46875 disk.drbd1289.in‚0 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.sda.out‚ 182.64453125) proc.memory.percent‚17.17219352722168 network.sent‚ 9.864524e+06 disk.rbd1.out‚ 14.9423828125# disk.nvme0n1.out‚3952.361328125 disk.drbd1356.out‚0 disk.drbd1965.in‚0 proc.cpu.threads‚29 disk.drbd1673.out‚0 disk.drbd1356.in‚0% memory_percent‚41.248369216918945 cpu‚13.735559758293537 disk.dm-0.out‚3952.361328125 disk.sda.in‚ 11.6328125 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0 disk.drbd2118.out‚0 disk.sda2.out‚ 182.64453125 disk.drbd1552.out‚0 disk.drbd1584.out‚0 disk.drbd2041.in‚0$ disk./.usageGB‚589.3504524230957 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd2078.in‚0 disk.drbd1828.in‚0 disk.drbd1938.in‚0 disk.nvme0n1.in‚440.0625 disk.drbd1550.in‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.dm-0.in‚440.0625 disk.drbd1510.in‚0 disk.drbd1226.out‚0 disk.drbd1550.out‚0! proc.memory.rssMB‚ 11253.96875 disk.drbd1226.in‚0 network.recv‚628029( disk./.usagePercent‚32.9595625205487 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.drbd1931.out‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 11.6328125 disk.drbd2041.out‚0 disk.drbd1673.in‚0¿Â"ù‘Û‚(:† ĽÅÓò›×ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚130.488' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚43.496 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0©•ŒaÜ‚(:Ö  ĽÅÓؠ⃠network.recv‚629612 disk.drbd1584.in‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0) disk./.usagePercent‚32.95956358723791 disk.sda.in‚ 11.6328125 disk.dm-0.in‚440.0625 disk.drbd2111.out‚0 disk.drbd1510.out‚0 network.sent‚ 9.909565e+06# disk./.usageGB‚589.350471496582 disk.drbd1550.out‚0 disk.rbd0.out‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0 disk.sda2.in‚ 11.6328125! proc.memory.rssMB‚ 11253.96875 cpu‚13.735221163078 disk.drbd1931.out‚0 disk.nvme0n1.in‚440.0625 disk.drbd1965.out‚0 disk.drbd2118.out‚0 disk.drbd1817.in‚0 disk.drbd1226.out‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0 disk.drbd1673.in‚0 disk.drbd1938.in‚0 disk.drbd1938.out‚0$ memory_percent‚41.24798774719238' proc.memory.availableMB‚ 38503.71875# disk.nvme0n1.out‚3952.560546875 disk.drbd1584.out‚0 disk.drbd2041.out‚0 disk.drbd1673.out‚0 disk.rbd1.in‚0) proc.memory.percent‚17.17219352722168 disk.sda.out‚ 182.92578125 disk.drbd1226.in‚0 disk.drbd1550.in‚0 disk.drbd1817.out‚0 disk.drbd1931.in‚0 disk.drbd2041.in‚0 disk.drbd1356.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 14.9423828125 disk.drbd1965.in‚0 disk.drbd1285.in‚0 disk.dm-0.out‚3952.560546875 disk.sda1.out‚0 disk.drbd1356.out‚0 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.sda2.out‚ 182.92578125 proc.cpu.threads‚29€U/y2Ý- _wandbruntime‚4470  _runtime‚4470—¶#/ÊÞÄ % _timestamp‚1.7858147254064806e+09 _runtime‚4470.598618449 _step‚848 ' lr-AdamW/pg1‚0.00027444801388202465 ' lr-AdamW/pg2‚0.00027444801388202465 trainer/global_step‚10649ü)3õÏßÉ trainer/global_step‚10649 % _timestamp‚1.7858147254064806e+09 _runtime‚4470.598618449 _step‚848 ' lr-AdamW/pg1‚0.00027444801388202465 ' lr-AdamW/pg2‚0.00027444801388202465ÐÄÃÒu2à- _wandbruntime‚4470  _runtime‚4470©ã} Ö'áÐO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚192201429342208 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚182830568715264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9624119258112 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚183261618254848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 112461075 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚192885737512960 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚182830551675904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41931445 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9370860626944 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9370860626944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚183261643158528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚192201324873216 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70752527 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30510283 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 $ _timestamp‚1.785814725407149e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70752529 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚178287379885472 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9347925406748 trainer/global_step‚10649 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚182830568715264 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 _runtime‚4470.598694618 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30510538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚178287396924832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 112461075 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9624252029440 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚192201324873216 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9370773197312 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚72441985 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9347839603624,àú âÕO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70752527 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 24888671c=Þá2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30510283 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 $ _timestamp‚1.785814725407149e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70752529 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚178287379885472 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9347925406748 trainer/global_step‚10649 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚182830568715264 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 _runtime‚4470.598694618 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30510538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚178287396924832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 112461075 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9624252029440 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 132771328 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚192201324873216 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9370773197312 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚72441985 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9347839603624 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚192201429342208 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚182830568715264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9624119258112 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚183261618254848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 112461075 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 157675008 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚192885737512960 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚182830551675904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41931445 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9370860626944 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9370860626944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚255 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚183261643158528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚192201324873216Ñ°´K?2ã- _wandbruntime‚4470  _runtime‚4470wPíà&äÚM REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30513135 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70759172 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚187652837134638 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 183240440 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 183230768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚192219371255808 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚192904004018176 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9371652816896 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚192219475731968 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚192903842156032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚178304137097152 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41935384 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 112471598 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70759170 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚178304120057792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9348802880082 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9371740253696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72448774 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚183278825371648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70759172 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30513390 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72448517 trainer/global_step‚10649 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚183278850275328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚182847735478272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 183240440 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚182847735478272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚182847718438912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _step‚850 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 112481268 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18‰÷Då&åßM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚182847735478272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚182847718438912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 _step‚850 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 112481268 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30513135 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70759172 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚187652837134638 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 183240440 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 183230768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚192219371255808 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚192904004018176 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9371652816896 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚192219475731968 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚192903842156032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚178304137097152 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41935384 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 112471598 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70759170 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚178304120057792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9348802880082 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9371740253696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72448774 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚183278825371648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70759172 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30513390 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72448517 trainer/global_step‚10649 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚183278850275328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚182847735478272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 183240440 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856Ò,-àà2æ- _wandbruntime‚4470  _runtime‚4470§)8­ çƒ 4loss/FBt2/assignment_loss‚1.5099839979863344e-07 /loss/FBt1/detection_loss‚1.0591236657152574 .loss/FRt1/detection_loss‚2.452932163527826 0loss/SRqqt1/detection_loss‚1.266439659333154 /loss/FBt2/detection_loss‚0.8347755461931229 trainer/global_step‚10649 % _timestamp‚1.7858147257573392e+09 0loss/FRt1/assignment_loss‚2.6328144277014385 /loss/FRt2/assignment_loss‚2.470136603156289 2loss/SRqqt2/assignment_loss‚0.6731828684874234 /loss/SRqqt2/detection_loss‚1.37801303072936 &loss/total_loss‚16.106436106862773 _runtime‚4470.947111516 _step‚851 2loss/SRqqt1/assignment_loss‚0.8229899980683475 3loss/FBt1/assignment_loss‚5.414081387655756e-07 /loss/FRt2/detection_loss‚2.5160274515440175 epoch‚0}.éèˆ 0loss/FRt1/assignment_loss‚2.6328144277014385 /loss/FRt2/assignment_loss‚2.470136603156289 2loss/SRqqt2/assignment_loss‚0.6731828684874234 /loss/SRqqt2/detection_loss‚1.37801303072936 &loss/total_loss‚16.106436106862773 _runtime‚4470.947111516 _step‚851 2loss/SRqqt1/assignment_loss‚0.8229899980683475 3loss/FBt1/assignment_loss‚5.414081387655756e-07 /loss/FRt2/detection_loss‚2.5160274515440175 epoch‚0 4loss/FBt2/assignment_loss‚1.5099839979863344e-07 /loss/FBt1/detection_loss‚1.0591236657152574 .loss/FRt1/detection_loss‚2.452932163527826 0loss/SRqqt1/detection_loss‚1.266439659333154 /loss/FBt2/detection_loss‚0.8347755461931229 trainer/global_step‚10649 % _timestamp‚1.7858147257573392e+09Ó y2!é‚(:’ Ó½ÅÓ³Ûå‚ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.384' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚42.461333333333336 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ©Š_ê‚(:Ô  Ó½ÅÓ™¸®ƒ( disk./.usagePercent‚32.9595625205487 disk.rbd1.out‚ 14.9423828125 disk.nvme0n1.in‚440.0625 disk.drbd2041.in‚0 disk.drbd1965.out‚0 disk.dm-0.in‚440.0625 disk.drbd1817.out‚0 disk.dm-0.out‚3952.771484375) proc.memory.percent‚17.17219352722168 disk.drbd1938.in‚0# disk.nvme0n1.out‚3952.771484375 disk.drbd1931.out‚0 disk.drbd1285.out‚0 disk.rbd1.in‚0 disk.drbd1584.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1510.in‚0 cpu‚13.73505258700841 disk.drbd2078.out‚0 disk.drbd2041.out‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.drbd1226.out‚0 disk.drbd1673.in‚0! proc.memory.rssMB‚ 11253.96875 disk.drbd1356.in‚0 disk.drbd1550.in‚0$ disk./.usageGB‚589.3504524230957 disk.sda.out‚ 183.390625 disk.drbd1510.out‚0 disk.drbd2078.in‚0 network.recv‚630554 disk.drbd1931.in‚0 disk.drbd2118.in‚0 disk.drbd1356.out‚0 disk.sda.in‚ 11.6328125 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1673.out‚0 disk.drbd1965.in‚0 disk.rbd0.out‚0 disk.drbd2118.out‚0 proc.cpu.threads‚29 disk.drbd1552.in‚0 disk.sda1.out‚0 disk.drbd1584.out‚0 disk.drbd1552.out‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 11.6328125& proc.memory.availableMB‚ 38503.4375 network.sent‚ 9.912683e+06 disk.sda2.out‚ 183.390625 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0% memory_percent‚41.248416900634766@ó€Á2ë- _wandbruntime‚4491  _runtime‚4491ÞYÈì _step‚852 & lr-AdamW/pg1‚0.0002757366232062899 & lr-AdamW/pg2‚0.0002757366232062899 trainer/global_step‚10699 % _timestamp‚1.7858147458253014e+09 _runtime‚4491.015231551Þ·?ýÍíÇ & lr-AdamW/pg2‚0.0002757366232062899 trainer/global_step‚10699 % _timestamp‚1.7858147458253014e+09 _runtime‚4491.015231551 _step‚852 & lr-AdamW/pg1‚0.0002757366232062899ÔÜj+2î- _wandbruntime‚4491  _runtime‚4491n|à ˆïÑO H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚179124388501472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚183688906865664 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚193791279333376 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71084679 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9414769332736 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9669305238528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inact8n…ÌO ive_split_bytes.small_pool.current‚ 134868480 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9391813870256 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚193791439105536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71084677 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚193103659159040 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9414769332736 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚188516116568604 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9414856762368 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 % _timestamp‚1.7858147458262033e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚193103763628032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42128295 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _runtime‚4491.019643655 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚193103659159040 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 trainer/global_step‚10699 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9414856762368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚184121998998528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚183688889826304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚183688906865664 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚183688889826304 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚72781992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚184121974094848 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚188516219411088 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 112989077 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 112998733 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9391728067132 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9669440107008 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚72782244 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 184073754 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 112998733 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30653697 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚179124405540832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42128297 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42128297 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚179124388501472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚183688906865664 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚193791279333376 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71084679 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9414769332736 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9669305238528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9391813870256 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚193791439105536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71084677 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚193103659159040 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9414769332736 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚188516116568604 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9414856762368 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 % _timestamp‚1.7858147458262033e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚193103763628032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42128295 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 _runtime‚4491.019643655 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚193103659159040 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 trainer/global_step‚10699 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9414856762368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚184121998998528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚183688889826304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚183688906865664 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚183688889826304 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚72781992ÕD=2ñ- _wandbruntime‚4491  _runtime‚4491?tá&òÛM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72788765 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42132232 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9392691343590 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚183706073628672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9415648952320 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚193121705541632 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9670342410240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9670209646080 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42132234 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚184139206115328 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 112999600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚183706056589312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71091320 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚188533734214146 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9415736389120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 % _timestamp‚1.7858147461682508e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 113009270 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚183706073628672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚854 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚183706056589312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72789025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚179141128673792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚193809548525568 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9415736389120 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71091322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 184090920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 trainer/global_step‚10699 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 184090920 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 184100592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚188533837056742 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚179141145713152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9392605540354 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71091320 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚193121810017792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚193809390857728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _runtime‚4491.358533384 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 184100592 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640lV‚ןóàM D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71091320 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚188533734214146 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9415736389120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 % _timestamp‚1.7858147461682508e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 113009270 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚258 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚183706073628672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚854 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚183706056589312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72789025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚179141128673792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚193809548525568 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9415736389120 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71091322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 184090920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 trainer/global_step‚10699 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 184090920 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 184100592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚188533837056742 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚179141145713152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9392605540354 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71091320 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚193121810017792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚193809390857728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 _runtime‚4491.358533384 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 184100592 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72788765 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42132232 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9392691343590 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚183706073628672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 157667840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9415648952320 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚193121705541632 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9670342410240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9670209646080 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42132234 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚184139206115328 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 112999600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚183706056589312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121Ö؃²â2ô- _wandbruntime‚4491  _runtime‚4491Y#‘ õ‡ /loss/FRt1/detection_loss‚2.3049095533788204 4loss/FBt2/assignment_loss‚1.5911769250217969e-06 &loss/total_loss‚15.128940316696257 epoch‚0 $ _timestamp‚1.785814746169459e+09 _runtime‚4491.359260558 0loss/FRt2/assignment_loss‚2.5863367742802725 2loss/SRqqt1/assignment_loss‚0.7733039988114485 1loss/SRqqt2/detection_loss‚1.0471715514780953 /loss/FBt1/detection_loss‚0.8405993301255096 /loss/FBt2/detection_loss‚1.1429951950381307 trainer/global_step‚10699 /loss/FRt1/assignment_loss‚2.103415920620873 /loss/FRt2/detection_loss‚2.3805632151394005 1loss/SRqqt1/detection_loss‚1.0056437645385514 _step‚855 2loss/SRqqt2/assignment_loss‚0.9439979958697222 4loss/FBt1/assignment_loss‚1.4262385088820956e-06£¡ÉöŒ /loss/FRt1/detection_loss‚2.3049095533788204 4loss/FBt2/assignment_loss‚1.5911769250217969e-06 &loss/total_loss‚15.128940316696257 epoch‚0 $ _timestamp‚1.785814746169459e+09 _runtime‚4491.359260558 0loss/FRt2/assignment_loss‚2.5863367742802725 2loss/SRqqt1/assignment_loss‚0.7733039988114485 1loss/SRqqt2/detection_loss‚1.0471715514780953 /loss/FBt1/detection_loss‚0.8405993301255096 /loss/FBt2/detection_loss‚1.1429951950381307 trainer/global_step‚10699 /loss/FRt1/assignment_loss‚2.103415920620873 /loss/FRt2/detection_loss‚2.3805632151394005 1loss/SRqqt1/detection_loss‚1.0056437645385514 _step‚855 2loss/SRqqt2/assignment_loss‚0.9439979958697222 4loss/FBt1/assignment_loss‚1.4262385088820956e-06×”w Äf÷‚(:Û  â½ÅÓ±­ƒ disk.drbd2041.in‚0 disk.drbd1828.in‚0 disk.drbd2111.out‚0 disk.dm-0.in‚440.0625# disk.nvme0n1.out‚3953.072265625 disk.rbd1.in‚0 disk.drbd2111.in‚0 network.recv‚632073 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd1550.out‚0 disk.drbd2078.out‚0 disk.drbd1552.in‚0 disk.rbd1.out‚ 14.9423828125) proc.memory.availableMB‚ 38503.5703125 disk.drbd1828.out‚0 disk.drbd1931.out‚0 disk.sda.in‚ 11.6328125 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.rbd0.out‚0 network.sent‚ 9.957728e+06 disk.drbd1931.in‚0 disk.dm-0.out‚3953.072265625 disk.drbd1226.in‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0 disk.drbd1584.in‚0) proc.memory.percent‚17.17219352722168( disk./.usagePercent‚32.9595625205487 disk.drbd1965.in‚0 disk.drbd1552.out‚0! proc.memory.rssMB‚ 11253.96875 cpu‚13.734937415091842 disk.drbd2078.in‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚440.0625 disk.sda1.in‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd1938.in‚0 disk.drbd1289.out‚0 disk.sda2.out‚ 183.88671875 disk.drbd1356.in‚0 disk.drbd1356.out‚0 disk.drbd1285.out‚0 disk.drbd1510.in‚0 disk.sda1.out‚0 disk.drbd1938.out‚0$ memory_percent‚41.24821424484253$ disk./.usageGB‚589.3504524230957 disk.drbd1673.out‚0 disk.drbd1584.out‚0 disk.sda.out‚ 183.88671875 disk.drbd1510.out‚0 disk.rbd0.in‚0 disk.drbd2118.in‚0 disk.sda2.in‚ 11.6328125 disk.drbd1285.in‚0 proc.cpu.threads‚29G˜Ëbœø‚(:‘ â½ÅÓ™úÚƒ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚99.387' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚33.129000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Óú?g2ù- _wandbruntime‚4511  _runtime‚4511aVÊÇúÁ & lr-AdamW/pg1‚0.0002770252325305553 & lr-AdamW/pg2‚0.0002770252325305553 trainer/global_step‚10749 $ _timestamp‚1.785814766797172e+09 _runtime‚4511.987105016 _step‚856Ï­9ÌûÆ & lr-AdamW/pg1‚0.0002770252325305553 & lr-AdamW/pg2‚0.0002770252325305553 trainer/global_step‚10749 $ _timestamp‚1.785814766797172e+09 _runtime‚4511.987105016 _step‚856ØOc”¸2ü- _wandbruntime‚4511  _runtime‚4511}§¶¨×'ýÑO JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚189397116490596 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42325147 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚71416829 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 113526735 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚265 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚184547227976704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 184943564 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30798130 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9714618056704 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _step‚857 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42325145 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9458765468160 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚194005993444864 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚184547227976704 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 113517079 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚10749 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71416829 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30797867 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 113517079 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚179961397117472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7858147667981076e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚184982329934848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚194006097913856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚73123012 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚194005993444864 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚184547245016064 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚194696972895232 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚189397013648112 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚179961414156832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 184933906 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 113526735 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9714476896768 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71416827 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9458852897792 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130-’R7Ü'þÖO ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚10749 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71416829 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30797867 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 113517079 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚179961397117472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7858147667981076e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚184982329934848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚194006097913856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚73123012 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚194005993444864 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚184547245016064 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚194696972895232 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚189397013648112 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚179961414156832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 184933906 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 113526735 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9714476896768 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71416827 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9458852897792 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 141159936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚189397116490596 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42325147 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚71416829 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 113526735 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚265 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚184547227976704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 184943564 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30798130 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9714618056704 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 _step‚857 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42325145 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9458765468160 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚194005993444864 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚184547227976704 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 113517079ÙK­Ä2ÿ- _wandbruntime‚4512  _runtime‚4512Où˜ð( €ÙM WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 184951072 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚184999537051648 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30800724 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 $ _timestamp‚1.785814767150805e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71423472 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 113527602 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9715378242560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 113537272 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 184960744 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9436494003862 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚184564411779072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚189414631293654 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9459645087744 NADeviceSt kü·atsMonitor.on_train_batch_end/allocation.large_pool.freed‚71423470 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9436579807098 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9459732524544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71423472 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚184564394739712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 184960744 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42329084 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚194714915294208 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚189414734136250 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚184564394739712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚73130061 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73129806 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9459732524544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚184999561955328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚184564411779072 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9459645087744 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚179978137289792 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚194024144303616 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 113527602 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42329082 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71423470 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚858 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 113537272 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚194024144303616 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9715513103872 trainer/global_step‚10749 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30800977 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚194024039827456 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992/‡½ä&ÞM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚194024039827456 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 184951072 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚184999537051648 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30800724 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 $ _timestamp‚1.785814767150805e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71423472 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 113527602 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9715378242560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 113537272 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 184960744 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9436494003862 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚184564411779072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚255 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚189414631293654 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9459645087744 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71423470 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9436579807098 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9459732524544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71423472 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚184564394739712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 184960744 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42329084 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚194714915294208 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚189414734136250 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚184564394739712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚73130061 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73129806 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9459732524544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚184999561955328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚184564411779072 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9459645087744 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚179978137289792 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚194024144303616 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 113527602 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42329082 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 134861312 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71423470 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚858 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 113537272 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚194024144303616 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9715513103872 trainer/global_step‚10749 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30800977 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121Úä ú2‚- _wandbruntime‚4512  _runtime‚4512©¦›ÿ ƒ† 2loss/SRqqt1/assignment_loss‚0.9499663628995991 2loss/SRqqt2/assignment_loss‚0.7821072286579949 1loss/SRqqt1/detection_loss‚1.1079727321719894 1loss/SRqqt2/detection_loss‚0.8706450381626686 /loss/FBt1/detection_loss‚0.8122324342506805 _runtime‚4512.341803739 _step‚859 /loss/FRt2/assignment_loss‚2.865803820522208 3loss/FBt1/assignment_loss‚8.829116166048883e-06 /loss/FRt2/detection_loss‚2.6541915300645327 /loss/FBt2/detection_loss‚0.9519983742099541 0loss/FRt1/assignment_loss‚2.4381866194307804 4loss/FBt2/assignment_loss‚0.00010023713644358168 epoch‚0 trainer/global_step‚10749 .loss/FRt1/detection_loss‚2.509755533002317 &loss/total_loss‚15.942968739625334 % _timestamp‚1.7858147671520016e+09œ]ü„‹ 0loss/FRt1/assignment_loss‚2.4381866194307804 4loss/FBt2/assignment_loss‚0.00010023713644358168 epoch‚0 trainer/global_step‚10749 .loss/FRt1/detection_loss‚2.509755533002317 &loss/total_loss‚15.942968739625334 % _timestamp‚1.7858147671520016e+09 2loss/SRqqt1/assignment_loss‚0.9499663628995991 2loss/SRqqt2/assignment_loss‚0.7821072286579949 1loss/SRqqt1/detection_loss‚1.1079727321719894 1loss/SRqqt2/detection_loss‚0.8706450381626686 /loss/FBt1/detection_loss‚0.8122324342506805 _runtime‚4512.341803739 _step‚859 /loss/FRt2/assignment_loss‚2.865803820522208 3loss/FBt1/assignment_loss‚8.829116166048883e-06 /loss/FRt2/detection_loss‚2.6541915300645327 /loss/FBt2/detection_loss‚0.9519983742099541Û ˆ³œ…‚(:‘ ñ½ÅÓ›÷¼ƒ gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚38.0 gpu.0.powerWatts‚98.612' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚32.870666666666665 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0&ƒøËa†‚(:Ö  ñ½ÅÓÜÆ˃ disk.sda2.in‚ 11.6328125$ disk./.usageGB‚589.3504524230957 disk.drbd1673.out‚0 disk.rbd1.in‚0 disk.drbd1226.in‚0 cpu‚13.734657640948125 disk.drbd1285.in‚0 disk.rbd0.in‚0 disk.drbd2118.in‚0 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1552.out‚0 proc.cpu.threads‚29 disk.drbd1965.out‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0 disk.drbd1938.out‚0 disk.sda2.out‚ 184.359375 disk.rbd1.out‚ 14.9423828125 disk.drbd1584.in‚0( disk./.usagePercent‚32.9595625205487 disk.drbd2078.in‚0 disk.drbd1828.out‚0 disk.drbd1552.in‚0 disk.drbd2118.out‚0 disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.drbd2078.out‚0 disk.drbd1510.in‚0# disk.nvme0n1.out‚3953.759765625 disk.drbd1356.out‚0 disk.drbd1550.out‚0 disk.dm-0.in‚440.0625 disk.drbd1817.out‚0 disk.nvme0n1.in‚440.0625 disk.drbd1584.out‚0 disk.dm-0.out‚3953.759765625 disk.drbd1931.in‚0 disk.sda.in‚ 11.6328125$ memory_percent‚41.24913215637207 network.sent‚ 1.0002671e+07 disk.rbd0.out‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.drbd1673.in‚0 disk.drbd1356.in‚0 disk.sda.out‚ 184.359375) proc.memory.percent‚17.17219352722168 disk.drbd2041.in‚0! proc.memory.rssMB‚ 11253.96875 disk.sda1.out‚0 disk.drbd1931.out‚0 disk.drbd1828.in‚0 disk.sda1.in‚0 disk.drbd1965.in‚0' proc.memory.availableMB‚ 38502.96875 disk.drbd1226.out‚0 disk.drbd1550.in‚0 network.recv‚633543 disk.drbd1289.in‚0up¨œ‡‚(:‘ €¾ÅÓ»éÛƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚120.142' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.04733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆhØjˆ‚(:ß  €¾ÅÓÃËåƒ$ disk./.usageGB‚589.3504524230957 disk.drbd2111.in‚0 disk.drbd1510.in‚0) proc.memory.percent‚17.17219352722168 disk.drbd1356.out‚0 disk.drbd2041.out‚0 network.sent‚ 1.0047717e+07 disk.drbd1584.in‚0 disk.drbd1226.in‚0 disk.drbd1285.out‚0 disk.drbd1828.in‚0" disk.dm-0.out‚3954.11669921875 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd2078.out‚0 cpu‚13.73443431282463 disk.sda1.in‚0 disk.drbd1584.out‚0 disk.rbd0.in‚0 disk.rbd1.out‚ 14.9423828125 disk.dm-0.in‚440.0625 disk.drbd1673.out‚0 disk.drbd1938.out‚0 disk.sda.out‚ 184.7578125 disk.drbd1289.in‚0 disk.drbd1817.in‚0 disk.drbd1938.in‚0 disk.drbd2041.in‚0 disk.drbd1965.out‚0 disk.drbd2078.in‚0 disk.sda2.in‚ 11.6328125* proc.memory.availableMB‚38502.80078125% disk.nvme0n1.out‚3954.11669921875 disk.drbd1356.in‚0 disk.drbd1931.out‚0 disk.rbd0.out‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.drbd1965.in‚0! proc.memory.rssMB‚ 11253.96875 proc.cpu.threads‚29( disk./.usagePercent‚32.9595625205487 disk.drbd1552.in‚0 disk.drbd1550.out‚0 disk.drbd1828.out‚0% memory_percent‚41.249388456344604 disk.drbd1510.out‚0 disk.sda2.out‚ 184.7578125 disk.sda.in‚ 11.6328125 disk.drbd1289.out‚0 disk.drbd2118.in‚0 disk.nvme0n1.in‚440.0625 disk.drbd1550.in‚0 disk.drbd1285.in‚0 disk.drbd1552.out‚0 disk.drbd1931.in‚0 disk.drbd1226.out‚0 network.recv‚635058qpÇ2‰- _wandbruntime‚4532  _runtime‚4532³‹ñvÉŠÃ ' lr-AdamW/pg1‚0.00027831384185482053 ' lr-AdamW/pg2‚0.00027831384185482053 trainer/global_step‚10799 % _timestamp‚1.7858147874620156e+09 _runtime‚ 4532.65195602 _step‚860vŠsJÎ‹È trainer/global_step‚10799 % _timestamp‚1.7858147874620156e+09 _runtime‚ 4532.65195602 _step‚860 ' lr-AdamW/pg1‚0.00027831384185482053 ' lr-AdamW/pg2‚0.00027831384185482053Üíéª2Œ- _wandbruntime‚4532  _runtime‚4532Ÿ ¶"ÑO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚248 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚195602487811072 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚185405583166464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9502761603584 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30941528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _step‚861 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 185794058 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚185405566127104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚194908432199680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30941774 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚73463523 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚185842710678528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚185842685774848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9759646458368 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 trainer/global_step‚10799 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858147874629285e+09 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9502849033216 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚185405566127104 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 185803716 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42521995 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚194908327730688 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 114054737 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚190278013570104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 114045081 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚190277910727620 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚180798405733472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.fr£ÜÖ!eed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚185405583166464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚180798422772832ÀËà9Ü'ŽÖO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71748977 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚194908327730688 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 114054737 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚190278013570104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 114045081 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚190277910727620 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 155577856 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚180798405733472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚185405583166464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚180798422772832 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚248 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚195602487811072 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚185405583166464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9502761603584 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30941528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 _step‚861 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 185794058 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚185405566127104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚194908432199680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30941774 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚73463523 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚185842710678528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚185842685774848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9759646458368 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 trainer/global_step‚10799 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7858147874629285e+09 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9502849033216 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚185405566127104 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 185803716 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42521995 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73470308 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚185859892891648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚10799 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 185820896 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 114055604 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚185422732890112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚185859917795328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚4533.009654779 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9480382467370 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚194926374113280 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71755620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 114065274 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 185820896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚180815162945152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 114065274 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9760543566848 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚190295528373162 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42525932 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9503728659968 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42525934 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚73470565 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 185811224 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚185422749929472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚185422749929472 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚195620598320640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9503641223168 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 185811224 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚190295631215758 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚185422732890112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚194926478589440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30944631 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30944376 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9503641223168 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9503728659968 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚1305|Ë´æ&‘àM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚185422732890112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚194926478589440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30944631 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30944376 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9503641223168 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9503728659968 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73470308 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚185859892891648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚10799 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 185820896 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 114055604 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚185422732890112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚185859917795328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 _runtime‚4533.009654779 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9480382467370 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚194926374113280 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71755620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚257 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 114065274 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 185820896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚180815162945152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 114065274 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9760543566848 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚190295528373162 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42525932 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9503728659968 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42525934 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚73470565 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 185811224 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚185422749929472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚185422749929472 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚195620598320640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9503641223168 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 185811224 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚190295631215758 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0Þ¶½ ¦2’- _wandbruntime‚4533  _runtime‚4533‹æ7Ø “… 1loss/SRqqt1/detection_loss‚1.1750035212062357 0loss/SRqqt2/detection_loss‚1.226628623163218 &loss/total_loss‚15.576719023705293 % _timestamp‚1.7858147878205783e+09 _runtime‚4533.010379101 _step‚863 0loss/FRt2/assignment_loss‚2.3644483745852605 4loss/FBt2/assignment_loss‚0.00028329932441077216 /loss/FBt1/detection_loss‚0.7273999691489293 0loss/FRt1/assignment_loss‚2.2637600627812473 2loss/SRqqt2/assignment_loss‚0.9423937954364673 3loss/FBt1/assignment_loss‚7.947245766252198e-07 .loss/FRt1/detection_loss‚2.291210376704582 .loss/FRt2/detection_loss‚2.511973992178712 /loss/FBt2/detection_loss‚0.8812951428375931 trainer/global_step‚10799 epoch‚0 2loss/SRqqt1/assignment_loss‚1.1923210716140602ïu7•”Š _step‚863 0loss/FRt2/assignment_loss‚2.3644483745852605 4loss/FBt2/assignment_loss‚0.00028329932441077216 /loss/FBt1/detection_loss‚0.7273999691489293 0loss/FRt1/assignment_loss‚2.2637600627812473 2loss/SRqqt2/assignment_loss‚0.9423937954364673 3loss/FBt1/assignment_loss‚7.947245766252198e-07 .loss/FRt1/detection_loss‚2.291210376704582 .loss/FRt2/detection_lossgÂÈ {‚2.511973992178712 /loss/FBt2/detection_loss‚0.8812951428375931 trainer/global_step‚10799 epoch‚0 2loss/SRqqt1/assignment_loss‚1.1923210716140602 1loss/SRqqt1/detection_loss‚1.1750035212062357 0loss/SRqqt2/detection_loss‚1.226628623163218 &loss/total_loss‚15.576719023705293 % _timestamp‚1.7858147878205783e+09 _runtime‚4533.010379101ßmMY³œ•‚(:‘ ¾ÅÓÖý¤ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚121.391' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.46366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0g¥– i–‚(:Þ  ¾ÅÓÝ×σ disk.drbd1356.out‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0 disk.drbd2118.out‚0 disk.drbd1673.in‚0( proc.memory.availableMB‚ 38502.703125 disk.rbd1.out‚ 15.0283203125 disk.drbd1510.out‚0 disk.drbd1938.in‚0$ memory_percent‚41.24953746795654 disk.sda.in‚ 11.6328125 disk.rbd0.out‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.drbd1965.out‚0 disk.drbd1285.in‚0 proc.cpu.threads‚29 disk.drbd2078.in‚0 disk.dm-0.in‚440.0625 disk.drbd2118.in‚0 disk.drbd2041.out‚0 disk.drbd1938.out‚0 disk.drbd1931.out‚0 disk.drbd2078.out‚0 disk.drbd1965.in‚0 disk.drbd1550.out‚0 disk.drbd1289.in‚0 disk.drbd1552.out‚0$ disk./.usageGB‚589.3628425598145 disk.rbd1.in‚0 disk.drbd1356.in‚0 disk.drbd1285.out‚0 disk.drbd1550.in‚0 disk.drbd1552.in‚0 network.sent‚ 1.0050756e+07 disk.drbd1931.in‚0 disk.sda.out‚ 185.36328125 disk.rbd0.in‚0) proc.memory.percent‚17.17219352722168 disk.drbd1226.in‚0 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1673.out‚0 network.recv‚635995 disk.drbd1584.in‚0 disk.drbd1226.out‚0! disk.dm-0.out‚3954.7294921875! proc.memory.rssMB‚ 11253.96875 cpu‚13.734323300717794 disk.sda2.in‚ 11.6328125 disk.sda2.out‚ 185.36328125 disk.drbd2111.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1817.in‚0 disk.drbd2041.in‚0) disk./.usagePercent‚32.96025544185578 disk.nvme0n1.in‚440.0625$ disk.nvme0n1.out‚3954.7294921875 disk.drbd1584.out‚0j†lÙ2—- _wandbruntime‚4553  _runtime‚4553_ë«Ç˜Á ' lr-AdamW/pg1‚0.00027960245117908584 ' lr-AdamW/pg2‚0.00027960245117908584 trainer/global_step‚10849 % _timestamp‚1.7858148082397604e+09 _runtime‚ 4553.429705 _step‚864§”¦%Ì™Æ ' lr-AdamW/pg1‚0.00027960245117908584 ' lr-AdamW/pg2‚0.00027960245117908584 trainer/global_step‚10849 % _timestamp‚1.7858148082397604e+09 _runtime‚ 4553.429705 _step‚864àèr §2š- _wandbruntime‚4553  _runtime‚4553coG×'›ÑO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚186263904277504 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72081129 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9804922811392 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9523393457656 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 114582739 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31085367 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31085103 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9546845168640 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 186663868 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚191158807807128 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚186263904277504 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9804787942912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚186263921316864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 114573083 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _runtime‚4553.431649247 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42718845 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚195810662016512 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚195810662016512 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9523479260780 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚195810766485504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚73804214 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9546757739008 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72081129 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚186263921316864 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚186703041614848 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9546845168640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚73803948 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚10849 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚266 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72081127 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42718847 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚196507989329920 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 114573083 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 % _timestamp‚1.7858148082406821e+09 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 114582739 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚195810766485504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72081127 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9546845168640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚73803948 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚10849 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚266 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72081127 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42718847 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚196507989329920 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 114573083 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 % _timestamp‚1.7858148082406821e+09 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 114582739 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚195810766485504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72081127 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚195810662016512 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚195810662016512 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9523479260780 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚195810766485504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚73804214 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9546757739008 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72081129 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚186263921316864 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚186703041614848 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 186671376 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚191176425452670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚181652171561152 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚186281071040512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 186671376 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚186720248731648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚195828812875264 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 114583606 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9547637358592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _runtime‚4553.782534408 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72087772 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚186281088079872 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42722782 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚186281071040512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9547724795392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9805690990592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9524270930878 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚10849 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 186681048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31088203 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 114593276 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚186720273635328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42722784 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚186281088079872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚181652154521792 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73810732 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚866 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚195828708399104 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72087770 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.fre*sJ®Šed‚72087770 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936ßÛ‚=æ&ŸàM M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31087950 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 186681048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚73810987 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚253 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 186671376 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚191176425452670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚181652171561152 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚186281071040512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 186671376 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚186720248731648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚195828812875264 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 114583606 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9547637358592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 _runtime‚4553.782534408 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 155570688 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72087772 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚186281088079872 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42722782 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚186281071040512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9547724795392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9805690990592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9524270930878 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 trainer/global_step‚10849 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 186681048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31088203 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 114593276 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚186720273635328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42722784 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚186281088079872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚181652154521792 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73810732 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚866 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚195828708399104 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72087770 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72087770 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0âW%·±2 - _wandbruntime‚4553  _runtime‚4553è4› ¡† epoch‚0 trainer/global_step‚10849 &loss/total_loss‚16.238425735151893 4loss/FBt2/assignment_loss‚3.9736418481576645e-08 /loss/FRt1/detection_loss‚2.6142587304115295 /loss/FRt2/detection_loss‚2.5794464629143476 3loss/FBt1/assignment_loss‚7.004153271227237e-06 1loss/SRqqt1/detection_loss‚1.2371919972144745 1loss/SRqqt2/detection_loss‚1.1234323248533267 % _timestamp‚1.7858148085934687e+09 _step‚867 0loss/FRt1/assignment_loss‚2.5595761285887826 0loss/FRt2/assignment_loss‚2.3749681003391743 2loss/SRqqt2/assignment_loss‚0.9439864677443568 -loss/FBt1/detection_loss‚1.16660823926715 /loss/FBt2/detection_loss‚0.7979533538843194 _runtime‚4553.783260501 2loss/SRqqt1/assignment_loss‚0.84099688604474072è¡¢‹ &loss/total_loss‚16.238425735151893 4loss/FBt2/assignment_loss‚3.9736418481576645e-08 /loss/FRt1/detection_loss‚2.6142587304115295 /loss/FRt2/detection_loss‚2.5794464629143476 3loss/FBt1/assignment_loss‚7.004153271227237e-06 1loss/SRqqt1/detection_loss‚1.2371919972144745 1loss/SRqqt2/detection_loss‚1.1234323248533267 % _timestamp‚1.7858148085934687e+09 _step‚867 0loss/FRt1/assignment_loss‚2.5595761285887826 0loss/FRt2/assignment_loss‚2.3749681003391743 2loss/SRqqt2/assignment_loss‚0.9439864677443568 -loss/FBt1/detection_loss‚1.16660823926715 /loss/FBt2/detection_loss‚0.7979533538843194 _runtime‚4553.783260501 2loss/SRqqt1/assignment_loss‚0.8409968860447407 epoch‚0 trainer/global_step‚10849ã^kU¸£‚(:’ ž¾ÅÓœìò‚ gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚133.533' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.510999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ë“9d¤‚(:Ù  ž¾ÅÓ­ƒÀƒ disk.drbd1931.in‚0 disk.rbd0.out‚0 disk.drbd1289.in‚0 disk.drbd1817.in‚0 disk.drbd1938.out‚0 disk.drbd1965.out‚0 disk.drbd1673.in‚0* disk./.usagePercent‚32.960393471438806 disk.sda2.out‚185.8125 disk.drbd1226.out‚0% disk.nvme0n1.out‚3957.42626953125 disk.drbd1550.in‚0 disk.drbd2078.in‚0" disk.dm-0.out‚3957.42626953125 disk.drbd1938.in‚0 disk.drbd2041.out‚0 disk.drbd2111.out‚0 disk.drbd1289.out‚0 disk.sda1.in‚0) proc.memory.percent‚17.17219352722168) proc.memory.availableMB‚ 38502.6640625 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.drbd1552.in‚0 disk.drbd2118.in‚0! proc.memory.rssMB‚ 11253.96875 proc.cpu.threads‚29 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.drbd2118.out‚0 disk.drbd1673.out‚0 disk.sda2.in‚ 11.6328125 disk.rbd1.in‚0 network.recv‚637681$ disk./.usageGB‚589.3653106689453 disk.nvme0n1.in‚440.0625 disk.drbd1356.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1550.out‚0 disk.dm-0.in‚440.0625 disk.drbd1510.out‚0 cpu‚13.73410454943992 disk.rbd0.in‚0 disk.drbd1817.out‚0 disk.drbd1584.out‚0 disk.drbd1226.in‚0 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.drbd1828.out‚0 disk.sda1.out‚0 disk.drbd1552.out‚0 disk.rbd1.out‚ 15.0283203125 disk.sda.out‚185.8125 disk.drbd2111.in‚0 disk.drbd1510.in‚0 disk.drbd1356.out‚0 disk.drbd1584.in‚0$ memory_percent‚41.24959707260132 network.sent‚ 1.0095815e+07ìÎÇ2¥- _wandbruntime‚4574  _runtime‚4574†ÖŠÊ¦Ä _runtime‚4574.485039782 _step‚868 ' lr-AdamW/pg1‚0.00028089106050335115 ' lr-AdamW/pg2‚0.00028089106050335115 trainer/global_step‚10899 % _timestamp‚1.7858148292950995e+09»%‰Ï§É % _timestamp‚1.7858148292950995e+09 _runtime‚4574.485039782 _step‚868 ' lr-AdamW/pg1‚0.00028089106050335115 ' lr-AdamW/pg2‚0.00028089106050335115 trainer/global_step‚10899䆂¹2¨- _wandbruntime‚4574  _runtime‚4574ñ}y×'©ÑO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚187122242427904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9590753874432 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31228404 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚182472440004832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚196713100771328 % _timestamp‚1.7858148292960527e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚182472422965472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚192039807729120 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72413277 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚187563397454848 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚196713100771328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42915695 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚197413560868352 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 115110741 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚187122242427904 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 115110741 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9567281921164 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72413277 trainer/global_step‚10899 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚196712996302336 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _runtime‚4574.488870675 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 115101085 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 187514362 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72413279 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚196712996302336 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9590841304064 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚187563422358528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚187122259467264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9590841304064 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 187524020 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42915697 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚187122259467264 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9850012029952 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9590753874432 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚187122259467264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9590841304064 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 187524020 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42915697 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚187122259467264 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9850012029952 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9590753874432 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚187122242427904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9590753874432 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31228404 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚182472440004832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚196713100771328 % _timestamp‚1.7858148292960527e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚182472422965472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚192039807729120 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72413277 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚187563397454848 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚196713100771328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42915695 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚254 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚197413560868352 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 115110741 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚187122242427904 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_trai„TîË n_batch_start/allocation.small_pool.allocated‚ 115110741 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚256 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9567281921164 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72413277 trainer/global_step‚10899 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚196712996302336 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 _runtime‚4574.488870675 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 115101085 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 187514362 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72413279 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚196712996302336 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9590841304064 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚187563422358528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 115121278 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚196731147161088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 187541200 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚197431674957824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9851045482496 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚870 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚192057322532178 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 187541200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9568245197622 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚187580629475328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9591633494016 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 187531528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72419920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9591720930816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 115121278 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚187139409190912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚187139409190912 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 115111608 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4574.835958277 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 115111608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74150870 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚74151107 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640ù#>œæ&­àM LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 115121278 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚196731147161088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 187541200 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚197431674957824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9851045482496 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚870 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚192057322532178 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 187541200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9568245197622 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚187580629475328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9591633494016 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 187531528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72419920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9591720930816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 115121278 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚187139409190912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚187139409190912 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 115111608 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4574.835958277 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 115111608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74150870 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚74151107 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640ænö°,2®- _wandbruntime‚4574  _runtime‚4574E¾> ¯ƒ .loss/FBt2/detection_loss‚0.802540310436771 epoch‚0 _runtime‚ 4574.83660186 4loss/FBt2/assignment_loss‚4.5413041031376146e-08 1loss/SRqqt1/detection_loss‚1.0014318373840276 &loss/total_loss‚15.209135621956676 trainer/global_step‚10899 $ _timestamp‚1.785814829646831e+09 0loss/FRt2/assignment_loss‚2.6602631990726175 2loss/SRqqt2/assignment_loss‚0.9540726724393499 /loss/FBt1/detection_loss‚0.5745602213708615 1loss/SRqqt1/assignment_loss‚0.761646498427297 4loss/FBt1/assignment_loss‚1.2009193160569911e-06 1loss/SRqqt2/detection_loss‚1.1486122482247763 _step‚871 /loss/FRt1/assignment_loss‚2.376536664439411 .loss/FRt1/detection_loss‚2.443816788494587 /loss/FRt2/detection_loss‚2.4856539353346214¬WW°ˆ trainer/global_step‚10899 $ _timestamp‚1.785814829646831e+09 0loss/FRt2/assignment_loss‚2.6602631990726175 2loss/SRqqt2/assignment_loss‚0.9540726724393499 /loss/FBt1/detection_loss‚0.5745602213708615 1loss/SRqqt1/assignment_loss‚0.761646498427297 4loss/FBt1/assignment_loss‚1.2009193160569911e-06 1loss/SRqqt2/detection_loss‚1.1486122482247763 _step‚871 /loss/FRt1/assignment_loss‚2.376536664439411 .loss/FRt1/detection_loss‚2.443816788494587 /loss/FRt2/detection_loss‚2.4856539353346214 .loss/FBt2/detection_loss‚0.802540310436771 epoch‚0 _runtime‚ 4574.83660186 4loss/FBt2/assignment_loss‚4.5413041031376146e-08 1loss/SRqqt1/detection_loss‚1.0014318373840276 &loss/total_loss‚15.209135621956676çŽrÝGœ±‚(:‘ ­¾ÅӋУƒ gpu.0.gpu‚43.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚132.215' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.07166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0­Mm h²‚(:Ý  ­¾ÅÓ›Üçƒ disk.drbd2078.in‚0 disk.drbd1673.in‚0 disk.drbd1226.out‚0 disk.drbd1289.out‚0 proc.cpu.threads‚29 disk.nvme0n1.in‚440.0625 disk.drbd2078.out‚0 disk.drbd2118.in‚0) proc.memory.availableMB‚ 38502.3203125$ disk./.usageGB‚589.3646659851074 disk.rbd1.in‚0 disk.drbd2041.in‚0 disk.drbd2041.out‚0 disk.drbd1965.out‚0 disk.drbd1828.in‚0 disk.drbd1938.out‚0 disk.drbd1550.out‚0 disk.drbd1938.in‚0$ disk.nvme0n1.out‚3959.7021484375 disk.sda.out‚ 188.3671875 disk.drbd1510.in‚0 disk.drbd1817.out‚0 disk.drbd1356.out‚0 disk.drbd1931.in‚0 disk.drbd1510.out‚0) proc.memory.percent‚17.17219352722168 disk.drbd1817.in‚0 disk.sda.in‚ 11.6328125 disk.drbd1285.in‚0 disk.drbd2111.out‚0 disk.sda2.in‚ 11.6328125 disk.rbd1.out‚ 15.0908203125 disk.rbd0.in‚0$ memory_percent‚41.25012159347534! proc.memory.rssMB‚ 11253.96875 disk.sda2.out‚ 188.3671875 disk.drbd1584.in‚0 disk.drbd1226.in‚0 disk.rbd0.out‚0 disk.drbd1285.out‚0 cpu‚13.733830621056297 network.sent‚ 1.0140826e+07 disk.drbd1584.out‚0 disk.drbd2111.in‚0 disk.drbd1552.in‚0 disk.drbd1356.in‚0 disk.drbd1550.in‚0 disk.dm-0.in‚440.0625 disk.drbd1673.out‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0 disk.drbd1931.out‚0 disk.drbd1828.out‚0 disk.drbd1552.out‚0 disk.sda1.in‚0 network.recv‚639373! disk.dm-0.out‚3959.7021484375 disk.sda1.out‚0) disk./.usagePercent‚32.96035741734371 disk.drbd1965.in‚0¬´ÞÁh³‚(:Ý  ¼¾ÅÓã€Âƒ disk.drbd1510.in‚0 disk.drbd1817.in‚0 network.recv‚640771 disk.sda1.out‚0 disk.drbd1673.out‚0 disk.drbd1584.in‚0 disk.sda.in‚ 11.6328125 disk.rbd0.out‚0$ disk./.usageGB‚589.3645668029785 disk.drbd1289.in‚0 disk.sda1.in‚0 disk.drbd2118.in‚0 disk.sda.out‚ 189.54296875 disk.drbd1965.out‚0% memory_percent‚41.249752044677734 disk.drbd1356.out‚0 disk.drbd2118.out‚0 disk.rbd1.in‚0& proc.memory.availableMB‚ 38502.5625 disk.nvme0n1.in‚440.0625$ disk.nvme0n1.out‚3960.8232421875 disk.drbd1285.in‚0 disk.drbd1931.out‚0 disk.drbd1550.out‚0) disk./.usagePercent‚32.96035187055984 disk.drbd2041.in‚0 disk.drbd1938.out‚0 disk.drbd1584.out‚0 disk.drbd1931.in‚0 disk.drbd1965.in‚0 disk.rbd0.in‚0 proc.cpu.threads‚29 disk.drbd1552.out‚0 disk.drbd1673.in‚0 disk.sda2.in‚ 11.6328125 disk.drbd1550.in‚0 disk.drbd1828.in‚0 disk.drbd1828.out‚0 disk.drbd1226.out‚0 network.sent‚ 1.0185815e+07 disk.dm-0.in‚440.0625 disk.drbd1817.out‚0 disk.sda2.out‚ 189.54296875 disk.drbd1226.in‚0 disk.drbd1285.out‚0) proc.memory.percent‚17.17219352722168 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd2041.out‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.drbd2078.out‚0! disk.dm-0.out‚3960.8232421875 disk.drbd2111.out‚0 disk.drbd1552.in‚0 disk.rbd1.out‚ 15.0908203125 disk.drbd1356.in‚0 disk.drbd2078.in‚0! proc.memory.rssMB‚ 11253.96875 cpu‚13.733642787586888yD¬ïœ´‚(:‘ ¼¾ÅÓÙ×½ƒ gpu.0.gpu‚27.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚127.475' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚42.49166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0—]2µ- _wandbruntime‚4595  _runtime‚4595­‘Ìï榀 % _timestamp‚1.7858148501259317e+09 _runtime‚4595.315872768 _step‚872 ' lr-AdamW/pg1‚0.00028217966982761647 ' lr-AdamW/pg2‚0.00028217966982761647 trainer/global_step‚10949Á›A”Ï·É _runtime‚4595.315872768 _step‚872 ' lr-AdamW/pg1‚0.00028217966982761647 ' lr-AdamW/pg2‚0.00028217966982761647 trainer/global_step‚10949 % _timestamp‚1.7858148501259317e+09èî~2¸- _wandbruntime‚4595  _runtime‚4595M(S~¹ÐO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚198318989227520 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9634837439488 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 159772160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚187980580578304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚187980597617664 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚187980580578304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚198319148999680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72745429 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚74483848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.sma+V™Xll_pool.freed‚ 115629087 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚197615330588160 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72745429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31371303 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9895370801152 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 trainer/global_step‚10949 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 115638743 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚197615435057152 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 115638743 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 115629087 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚183309448620832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31371563 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚197615435057152 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚187980597617664 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9634750009856 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9895235932672 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 188374514 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚192920601966144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚192920704808628 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚4595.320642884 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72745427 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚74484110 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43112547 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 188384172 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚188423753294848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚197615330588160 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72745429 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚74483848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 115629087 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚197615330588160 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72745429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31371303 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9895370801152 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 trainer/global_step‚10949 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 115638743 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚197615435057152 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 115638743 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 115629087 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚183309448620832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31371563 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 134868480 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚197615435057152 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚187980597617664 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9634750009856 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9895235932672 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 188374514 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚192920601966144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚192920704808628 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 _runtime‚4595.320642884 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72745427 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚74484110 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚260 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43112547 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 188384172 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚188423753294848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚197615330588160 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚198337256093184 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9612133661130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 188391680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72752072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚192938322454282 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚4595.672222334 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9635717066240 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚198337102619648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 115639610 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚188440960411648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 115649280 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74490651 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 trainer/global_step‚10949 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 188401352 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9635629629440 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚197633376970752 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9612047857894 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚183326171753792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚187997747341312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31374169 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚874 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72752070 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9635717066240 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 115649280 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31374423 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 $ _timestamp‚1.785814850481888e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43116484 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 188391680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚187997764380672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9896142208000 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 _step‚874 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72752070 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9635717066240 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 115649280 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31374423 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 $ _timestamp‚1.785814850481888e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43116484 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 188391680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚187997764380672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9896142208000 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚198337256093184 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9612133661130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 188391680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72752072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚192938322454282 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L=DeviceStatsMuG¿OMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚4595.672222334 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9635717066240 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚198337102619648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 115639610 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚188440960411648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 115649280 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74490651 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 trainer/global_step‚10949 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 188401352 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251êý#ë2¾- _wandbruntime‚4595  _runtime‚4595åà|ù¿ó /loss/FBt2/detection_loss‚0.8638110637043913 %loss/total_loss‚14.31015281081049 0loss/FRt2/assignment_loss‚2.3711530071933096 2loss/SRqqt1/assignment_loss‚0.7206892342104212 4loss/FBt1/assignment_loss‚1.2417630805098648e-07 0loss/FRt1/assignment_loss‚2.1195149557745974 0loss/FBt1/detection_loss‚0.49338827868147445 trainer/global_step‚10949 _step‚875 epoch‚0 1loss/SRqqt2/assignment_loss‚0.592866737712341 loss/FBt2/assignment_loss‚0 /loss/FRt1/detection_loss‚2.3285705291706584 /loss/FRt2/detection_loss‚2.5112312630545803 1loss/SRqqt1/detection_loss‚1.0580348520175271 _runtime‚4595.672935273 1loss/SRqqt2/detection_loss‚1.2508927651148822 % _timestamp‚1.7858148504831443e+09`š.…þÀø _runtime‚4595.672935273 1loss/SRqqt2/detection_loss‚1.2508927651148822 % _timestamp‚1.7858148504831443e+09 /loss/FBt2/detection_loss‚0.8638110637043913 %loss/total_loss‚14.31015281081049 0loss/FRt2/assignment_loss‚2.3711530071933096 2loss/SRqqt1/assignment_loss‚0.7206892342104212 4loss/FBt1/assignment_loss‚1.2417630805098648e-07 0loss/FRt1/assignment_loss‚2.1195149557745974 0loss/FBt1/detection_loss‚0.49338827868147445 trainer/global_step‚10949 _step‚875 epoch‚0 1loss/SRqqt2/assignment_loss‚0.592866737712341 loss/FBt2/assignment_loss‚0 /loss/FRt1/detection_loss‚2.3285705291706584 /loss/FRt2/detection_loss‚2.5112312630545803 1loss/SRqqt1/detection_loss‚1.0580348520175271ë;Á;}Á‚(:’ ˾ÅÓØ颃 gpu.0.gpu‚41.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚131.283' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚43.760999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¤Drwa‚(:Ö  ˾ÅÓþ‘¾ƒ disk.drbd1931.in‚0 disk.drbd2078.in‚0 disk.drbd1285.in‚0 disk.drbd1550.out‚0 disk.rbd0.in‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.drbd2118.out‚0 disk.rbd1.out‚ 15.123046875 disk.drbd2041.in‚0 disk.sda1.out‚0$ disk.nvme0n1.out‚3961.5419921875 disk.dm-0.in‚440.0625 disk.drbd1673.in‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 disk.sda.in‚ 11.6328125 disk.rbd0.out‚0% memory_percent‚41.249895095825195 proc.memory.rssMB‚11254 disk.drbd1828.in‚0 disk.sda1.in‚0 disk.sda2.in‚ 11.6328125 disk.drbd1226.out‚0 disk.drbd1285.out‚0' proc.memory.availableMB‚ 38502.46875 network.recv‚641848) disk./.usagePercent‚32.96035315058688 disk.drbd1510.out‚0 disk.drbd1673.out‚0 disk.drbd1356.out‚0 proc.cpu.threads‚29 disk.drbd2111.out‚0 disk.drbd2118.in‚0 disk.sda.out‚ 190.01953125 disk.drbd1817.out‚0 network.sent‚ 1.0188852e+07! disk.dm-0.out‚3961.5419921875 disk.drbd1938.in‚0 disk.drbd1584.in‚0 disk.drbd1584.out‚0 disk.drbd2078.out‚0 disk.drbd2111.in‚0 disk.drbd1817.in‚0 disk.drbd1931.out‚0 disk.drbd1965.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.nvme0n1.in‚440.0625 disk.drbd1226.in‚0 disk.drbd1965.in‚0 cpu‚13.733374203314067 disk.drbd1550.in‚0 disk.drbd1828.out‚0 disk.drbd2041.out‚0( proc.memory.percent‚17.1722412109375$ disk./.usageGB‚589.3645896911621 disk.drbd1289.out‚0 disk.sda2.out‚ 190.01953125 disk.drbd1356.in‚0é}§>2Ã- _wandbruntime‚4615  _runtime‚4615yÕ!ÈÄ & lr-AdamW/pg2‚0.0002834682791518818 trainer/global_step‚10999 % _timestamp‚1.7858148706841645e+09 _runtime‚4615.874094639 _step‚876 & lr-AdamW/pg1‚0.0002834682791518818ÒkÿÍÅÇ & lr-AdamW/pg1‚0.0002834682791518818 & lr-AdamW/pg2‚0.0002834682791518818 trainer/global_step‚10999 % _timestamp‚1.7858148706841645e+09 _runtime‚4615.874094639 _step‚876ìuä á2Æ- _wandbruntime‚4615  _runtime‚4615›§ÔÖ'ÇÐO aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚189284109134848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9940499943424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9940360880640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚74822809 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚184146457236832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.785814870685064e+09 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚10999 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9655144651304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚188838935768064 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚198517664873984 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 189244324 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73077577 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚193801499045652 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚198517664873984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚193801601888136 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚877 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9678746145280 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚74823063 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9678833574912 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚199224633981952 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 116166745 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚188838935768064 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9678746145280 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚188838918728704 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 116166745 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚199224470015488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚198517769342976 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9678833574912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0*©zjÛ'ÈÕO YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9940360880640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚74822809 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚184146457236832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.785814870685064e+09 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 163966464 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚10999 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9655144651304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚188838935768064 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚198517664873984 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 189244324 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚252 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73077577 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚193801499045652 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚198517664873984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚193801601888136 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 _step‚877 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9678746145280 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚74823063 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9678833574912 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚199224633981952 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 116166745 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚188838935768064 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚254 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9678746145280 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚188838918728704 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 116166745 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚199224470015488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚198517769342976 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9678833574912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚189284109134848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9940499943424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚189301316251648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚184163197409152 % _timestamp‚1.7858148710306156e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9679625764864 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚198535711256576 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚4616.220907201 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9679713201664 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43313334 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43313332 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73084222 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚10999 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚199242740608512 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9679713201664 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚193819116691194 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚199242580843520 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9679625764864 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚878 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73084220 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚188856102531072 OADeviceStatsMonitor.o<… ®“n_train_batch_end/allocation.small_pool.freed‚ 116167612 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚184163180369792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚189301341155328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚198535815732736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 116167612 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73084222 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚188856085491712 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 189261504 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 189251832 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 116177282 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31516211 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9655936321402 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 189261504 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚74829783 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74829543 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚198535815732736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73084220 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚238 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 116177282 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9941399453184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9941264591872 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚193819219533790 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9656022124638 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚188856085491712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43313334 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43313332 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73084222 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚10999 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚199242740608512 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9679713201664 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚193819116691194 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚199242580843520 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9679625764864 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚878 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73084220 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚188856102531072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 116167612 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚184163180369792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚189301341155328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚198535815732736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 116167612 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73084222 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚188856085491712 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 189261504 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 189251832 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 116177282 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31516211 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9655936321402 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 189261504 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚74829783 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74829543 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 159764992 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚198535815732736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73084220 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚238 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 116177282 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9941399453184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9941264591872 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚193819219533790 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9656022124638 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚188856085491712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚189301316251648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚184163197409152 % _timestamp‚1.7858148710306156e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9679625764864 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚198535711256576 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 _runtime‚4616.220907201 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9679713201664 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640î|ÍŠ³2Ì- _wandbruntime‚4616  _runtime‚4616 6Í 3loss/FBt1/assignment_loss‚3.809855978431642e-06 &loss/total_loss‚16.635759708856895 _runtime‚4616.221825804 .loss/FRt1/assignment_loss‚2.35421971604228 1loss/SRqqt1/detection_loss‚1.1485813027506486 1loss/SRqqt2/detection_loss‚1.1338789750808391 /loss/FRt2/assignment_loss‚2.502429424147857 3loss/FBt2/assignment_loss‚0.0005883256045098489 /loss/FBt2/detection_loss‚1.5807295157932317 _step‚879 2loss/SRqqt2/assignment_loss‚0.8080109396571482 .loss/FRt1/detection_loss‚2.576094985567033 /loss/FRt2/detection_loss‚2.6524335204770693 -loss/FBt1/detection_loss‚1.10217157907784 epoch‚0 trainer/global_step‚10999 % _timestamp‚1.7858148710320356e+09 2loss/SRqqt1/assignment_loss‚0.7766176148024561Å¥ Ά _step‚879 2loss/SRqqt2/assignment_loss‚0.8080109396571482 .loss/FRt1/detection_loss‚2.576094985567033 /loss/FRt2/detection_loss‚2.6524335204770693 -loss/FBt1/detection_loss‚1.10217157907784 epoch‚0 trainer/global_step‚10999 % _timestamp‚1.7858148710320356e+09 2loss/SRqqt1/assignment_loss‚0.7766176148024561 3loss/FBt1/assignment_loss‚3.809855978431642e-06 &loss/total_loss‚16.635759708856895 _runtime‚4616.221825804 .loss/FRt1/assignment_loss‚2.35421971604228 1loss/SRqqt1/detection_loss‚1.1485813027506486 1loss/SRqqt2/detection_loss‚1.1338789750808391 /loss/FRt2/assignment_loss‚2.502429424147857 3loss/FBt2/assignment_loss‚0.0005883256045098489 /loss/FBt2/detection_loss‚1.5807295157932317ï2œ¦(›Ï‚(: Ú¾ÅÓ¨…ƒ gpu.0.gpu‚42.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚96.469' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚32.15633333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0í€H_Ђ(:Ô  Ú¾ÅÓӖȃ disk.drbd1285.out‚0* proc.memory.availableMB‚38502.18359375$ disk./.usageGB‚589.3645896911621 disk.drbd1828.in‚0 disk.sda1.out‚0 proc.memory.rssMB‚11254( proc.memory.percent‚17.1722412109375 network.recv‚643417 disk.drbd1510.out‚0 disk.drbd1356.in‚0 disk.rbd0.in‚0 disk.drbd1552.out‚0 disk.drbd2118.in‚0 disk.drbd1584.in‚0 network.sent‚ 1.0233999e+07 disk.drbd1965.out‚0 disk.drbd1550.in‚0 disk.drbd2078.out‚0) disk./.usagePercent‚32.96035315058688 disk.drbd1828.out‚0 disk.drbd1673.out‚0 disk.drbd1938.in‚0 disk.sda2.out‚ 190.328125$ disk.nvme0n1.out‚3961.7763671875 disk.drbd1289.in‚0 disk.drbd1931.out‚0 disk.sda.in‚ 11.6328125 disk.drbd2118.out‚0 disk.drbd1356.out‚0 disk.drbd1226.out‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 15.123046875 disk.drbd1965.in‚0 disk.drbd1931.in‚0 proc.cpu.threads‚29 disk.drbd1550.out‚0 disk.drbd1510.in‚0 disk.drbd1938.out‚0 disk.sda2.in‚ 11.6328125% memory_percent‚41.250330209732056 disk.sda.out‚ 190.328125 disk.dm-0.in‚440.0625 disk.sda1.in‚0 disk.nvme0n1.in‚440.0625 disk.drbd2111.in‚0 disk.drbd2111.out‚0 disk.drbd1226.in‚0 disk.drbd1285.in‚0 disk.drbd2078.in‚0 disk.drbd1552.in‚0 cpu‚13.73329353026884! disk.dm-0.out‚3961.7763671875 disk.drbd2041.in‚0 disk.drbd1289.out‚0 disk.drbd1817.in‚0 disk.drbd1817.out‚0 disk.drbd1673.in‚0 disk.drbd1584.out‚0 disk.drbd2041.out‚0‘æ‘Ñ‚(:† é¾ÅÓçΛƒ gpu.0.gpu‚25.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚124.671' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚41.557 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Axmà`Ò‚(:Õ  é¾ÅÓæú߃ disk.drbd2118.in‚0 disk.drbd1584.out‚0 disk.drbd1510.in‚0 disk.rbd1.in‚0 disk.sda.in‚ 11.6328125% memory_percent‚41.250330209732056( proc.memory.percent‚17.1722412109375 disk.drbd1817.in‚0 disk.drbd1550.out‚0* proc.memory.availableMB‚38502.18359375 disk.sda1.in‚0$ disk./.usageGB‚589.3645706176758 disk.drbd1938.in‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd2118.out‚0 disk.sda2.in‚ 11.6328125 disk.drbd2078.out‚0 disk.drbd1285.in‚0 disk.drbd1673.in‚0 disk.drbd2041.in‚0 disk.drbd2078.in‚0 disk.drbd1828.in‚0 network.recv‚644905 disk.nvme0n1.in‚440.0625 disk.sda.out‚ 190.59375 disk.drbd1828.out‚0 disk.drbd2111.in‚0 disk.drbd1938.out‚0$ disk.nvme0n1.out‚3962.8369140625 disk.drbd1965.in‚0 disk.drbd1285.out‚0 disk.drbd1817.out‚0 cpu‚13.732999817565199 network.sent‚ 1.0278927e+07 disk.dm-0.in‚440.0625 disk.rbd1.out‚15.18603515625 disk.drbd1584.in‚0 disk.drbd2041.out‚0 disk.rbd0.in‚0 disk.drbd1289.out‚0 disk.drbd1552.in‚0 disk.sda2.out‚ 190.59375 disk.sda1.out‚0 disk.drbd2111.out‚0 disk.drbd1510.out‚0 disk.drbd1289.in‚0 disk.drbd1931.in‚0) disk./.usagePercent‚32.96035208389768 disk.drbd1226.in‚0 disk.drbd1356.in‚0 disk.rbd0.out‚0 proc.cpu.threads‚29! disk.dm-0.out‚3962.8369140625 disk.drbd1552.out‚0 disk.drbd1673.out‚0 disk.drbd1356.out‚0 disk.drbd1931.out‚0 proc.memory.rssMB‚11254 disk.drbd1965.out‚0ÞJF2Ó- _wandbruntime‚4636  _runtime‚4636‹…>ÅÔ¿ trainer/global_step‚11049 " _timestamp‚1.7858148913709e+09 _runtime‚4636.560844171 _step‚880 & lr-AdamW/pg1‚0.0002847568884761471 & lr-AdamW/pg2‚0.00028475688847614718}8îÊÕÄ _step‚880 & lr-AdamW/pg1‚0.0002847568884761471 & lr-AdamW/pg2‚0.0002847568884761471 trainer/global_step‚11049 " _timestamp‚1.7858148913709e+09 _runtime‚4636.560844171ð‡Gá™2Ö- _wandbruntime‚4636  _runtime‚4636µ™Ê”Ö'×ÐO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 116685091 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199419999159808 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚184983465852832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31656463 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 116694747 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚190144464974848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9985558129152 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73409729 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚194682498967644 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73409727 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9699033114812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75162963 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9722829710336 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9698947311688 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9722742280704 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚189697273918464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚184983448813472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚200130023104000 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚11049 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 116694747 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚189697256879104 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚200130176584704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73409729 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚190144489878528 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199419999159808 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚189697273918464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 190104476 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640kñÖK%ØÕO GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 116685091 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚255 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 $ _timestamp‚1.785814891371812e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9722829710336 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9698947311688 WEDeviceStatsMonitor.on_train_batch_sI­Ï¶$tart/active_bytes.small_pool.freed‚ 9722742280704 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚189697273918464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚184983448813472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚200130023104000 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚11049 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 116694747 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚189697256879104 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚200130176584704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73409729 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 153480704 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚190144489878528 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199419999159808 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚189697273918464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 190104476 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 116685091 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199419999159808 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚253 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚184983465852832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31656463 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 116694747 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚190144464974848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 128577024 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9985558129152 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73409729 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚194682498967644 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73409727 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9699033114812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75162963 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9723621900288 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 190111984 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73416370 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73416372 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚199438150018560 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚189714423642112 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75169542 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43510184 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚199438150018560 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9986461594624 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚194700013770702 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 116695614 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚199438045542400 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚11049 $ _timestamp‚1.785814891713224e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 116705284 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 190121656 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚200148133686272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚190161672091648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73416372 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 116695614 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43510182 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9986590164480 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚185000206025152 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9723709337088 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚189714423642112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9723709337088 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73416370 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 116705284 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚882 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚194700116613298 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9699910588146 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 190111984 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚200148287159808 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚75169801 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31659617 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚190161696995328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31659360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9699824784910X–¶!å&ÛßM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9723621900288 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 190111984 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚257 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73416370 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73416372 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚199438150018560 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚189714423642112 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75169542 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43510184 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚199438150018560 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9986461594624 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 153473536 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚194700013770702 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 116695614 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚199438045542400 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚11049 $ _timestamp‚1.785814891713224e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 116705284 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 190121656 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚200148133686272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚190161672091648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73416372 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 116695614 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43510182 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9986590164480 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 128569856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚259 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚185000206025152 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9723709337088 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚189714423642112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9723709337088 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73416370 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 116705284 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚882 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚194700116613298 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9699910588146 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 190111984 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚200148287159808 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚75169801 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31659617 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚190161696995328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31659360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9699824784910òþrÇý2Ü- _wandbruntime‚4636  _runtime‚4636×wâÀ ݆ &loss/total_loss‚15.296226448477888 0loss/FRt2/assignment_loss‚2.6957446863484935 4loss/FBt2/assignment_loss‚1.5549036222835086e-08 .loss/FRt1/detection_loss‚2.299525906096448 /loss/FRt2/detection_loss‚2.5552132448484732 1loss/SRqqt2/detection_loss‚1.2546549225782058 /loss/FBt1/detection_loss‚0.9695604408635214 /loss/FBt2/detection_loss‚0.7710344407950407 3loss/FBt1/assignment_loss‚5.054324296812259e-06 1loss/SRqqt1/detection_loss‚1.0580060393918012 _step‚883 epoch‚0 /loss/FRt1/assignment_loss‚2.008484741488656 trainer/global_step‚11049 _runtime‚4636.904159252 2loss/SRqqt1/assignment_loss‚0.8723269639630541 2loss/SRqqt2/assignment_loss‚0.8116699922308603 % _timestamp‚1.7858148917143862e+09‡Ìr“Þ‹ &loss/total_loss‚15.296226448477888 0loss/FRt2/assignment_loss‚2.6957446863484935 4loss/FBt2/assignment_loss‚1.5549036222835086e-08 .loss/FRt1/detection_loss‚2.299525906096448 /loss/FRt2/detection_loss‚2.5552132448484732 1loss/SRqqt2/detection_loss‚1.2546549225782058 /loss/FBt1/detection_loss‚0.9695604408635214 /loss/FBt2/detection_loss‚0.7710344407950407 3loss/FBt1/assignment_loss‚5.054324296812259e-06 1loss/SRqqt1/detection_loss‚1.0580060393918012 _step‚883 epoch‚0 /loss/FRt1/assignment_loss‚2.008484741488656 trainer/global_step‚11049 _runtime‚4636.904159252 2loss/SRqqt1/assignment_loss‚0.8723269639630541 2loss/SRqqt2/assignment_loss‚0.8116699922308603 % _timestamp‚1.7858148917143862e+09óiÏÈß‚(:„ ø¾ÅÓú³€ƒ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚99.954' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚33.318 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ö]Ž*6à‚(:Û  ø¾ÅÓϒà disk.sda.in‚ 11.6328125 disk.drbd2078.in‚0 disk.rbd0.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1550.out‚0 disk.drbd1285.out‚0 disk.dm-0.in‚440.0625 disk.drbd1938.in‚0% memory_percent‚41.249603033065796 disk.sda2.in‚ 11.6328125 disk.rbd1.in‚0 proc.memory.rssMB‚11254 disk.drbd1289.out‚0$ disk./.usageGB‚589.3645896911621 disk.drbd1931.out‚0 disk.drbd1673.in‚0 disk.nvme0n1.in‚440.0625 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.drbd1673.out‚0 disk.sda2.out‚ 191.08203125* proc.memory.availableMB‚38502.66015625( proc.memory.percent‚17.1722412109375 disk.sda.out‚ 191.08203125 disk.drbd1965.out‚0 disk.drbd1828.in‚0 disk.drbd1510.in‚0 network.sent‚ 1.0281985e+07 network.recv‚645773 disk.drbd1931.in‚0 disk.drbd1226.in‚0 disk.drbd1289.in‚0 disk.drbd1938.out‚0 disk.drbd1584.out‚0 disk.drbd1226.out‚0 disk.drbd1550.in‚0 disk.drbd2078.out‚0 disk.sda1.in‚0 disk.rbd1.out‚15.18603515625 disk.drbd1356.in‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1552.out‚0 disk.rbd0.in‚0 disk.drbd1828.out‚0 proc.cpu.threads‚29) disk./.usagePercent‚32.960’`9 035315058688$ disk.nvme0n1.out‚3963.0556640625 disk.drbd1817.out‚0 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.drbd1285.in‚0 disk.sda1.out‚0 disk.drbd2118.out‚0 disk.drbd1584.in‚0! disk.dm-0.out‚3963.0556640625 disk.drbd2111.out‚0 cpu‚13.732816611267383*M=C2á- _wandbruntime‚4657  _runtime‚4657ü’®µÊâÄ trainer/global_step‚11099 % _timestamp‚1.7858149121221814e+09 _runtime‚4657.312123476 _step‚884 ' lr-AdamW/pg1‚0.00028604549780041234 ' lr-AdamW/pg2‚0.00028604549780041234<µ1–ÏãÉ ' lr-AdamW/pg2‚0.00028604549780041234 trainer/global_step‚11099 % _timestamp‚1.7858149121221814e+09 _runtime‚4657.312123476 _step‚884 ' lr-AdamW/pg1‚0.00028604549780041234ô¶Ô–œ2ä- _wandbruntime‚4657  _runtime‚4657~Ø0Ø'åÒO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 117222749 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201035588266496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚191004820814848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75502681 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚200322333445632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190555612068864 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 117222749 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9742835775196 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚190555595029504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚200322333445632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9766825845760 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 117213093 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9766738416128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 190964628 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚191004845718528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31799584 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 117213093 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 190954970 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73741877 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚200322437914624 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31799334 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚195563396047152 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73741879 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 $ _timestamp‚1.785814912123106e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10030889737216 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9766825845760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚200322437914624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚190555595029504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚190555612068864 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚195563293204668 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 117222749 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201035588266496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚191004820814848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75502681 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 122285568 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚200322333445632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190555612068864 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 117222749 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9742835775196 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚190555595029504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚200322333445632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9766825845760 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 117213093 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9766738416128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 190964628 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚191004845718528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31799584 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 117213093 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 190954970 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73741877 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚200322437914624 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31799334 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚195563396047152 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73741879 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 $ _timestamp‚1.785814912123106e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10030889737216 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚250 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9766825845760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚200322437914624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚190555595029504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 3021996032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚190555612068864 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚195563293204668 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚190572778831872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 117233286 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚75509526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚200340379828224 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75509274 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚200340484304384 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31802492 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73748522 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201053699381248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 190981808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 117223616 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 $ _timestamp‚1.785814912472877e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31802242 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚195580910850210 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚195581013692806 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9743799051654 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 190972136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 117233286 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9743713248418 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73748520 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚191022052835328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚4657.663177355 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 190972136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚200340484304384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 117223616 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚190572761792512 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10031808408064 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10031671449600 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9767705472512 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9767705472512 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚190572778831872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 117233286 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚75509526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚200340379828224 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75509274 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚200340484304384 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31802492 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73748522 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201053699381248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 190981808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 117223616 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 $ _timestamp‚1.785814912472877e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31802242 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚195580910850210 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚195581013692806 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9743799051654 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 190972136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 117233286 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9743713248418 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73748520 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚191022052835328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚4657.663177355 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 190972136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚200340484304384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚250 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 117223616 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚190572761792512 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10031808408064ö@WC¨2ê- _wandbruntime‚4657  _runtime‚4657¥2iôòëì /loss/FRt2/detection_loss‚2.3868139173756253 $ _timestamp‚1.785814912474068e+09 _runtime‚4657.663848045 /loss/FRt2/assignment_loss‚2.480785298220655 2loss/SRqqt1/assignment_loss‚0.9448958300937224 1loss/SRqqt2/assignment_loss‚0.743253170397796 loss/FBt2/assignment_loss‚0 .loss/FBt1/detection_loss‚0.810977195006083 .loss/FBt2/detection_loss‚1.733330658947428 3loss/FBt1/assignment_loss‚4.733298563554827e-07 1loss/SRqqt1/detection_loss‚1.2943551399391786 1loss/SRqqt2/detection_loss‚1.0081356468368066 epoch‚0 trainer/global_step‚11099 %loss/total_loss‚16.02750418915676 /loss/FRt1/assignment_loss‚2.130275881909705 /loss/FRt1/detection_loss‚2.4946809770999017 _step‚887©MŒ÷ìñ /loss/FRt2/assignment_loss‚2.480785298220655 2loss/SRqqt1/assignment_loss‚0.9448958300937224 1loss/SRqqt2/assignment_loss‚0.743253170397796 loss/FBt2/assignment_loss‚0 .loss/FBt1/detection_loss‚0.810977195006083 .loss/FBt2/detection_loss‚1.733330658947428 3loss/FBt1/assignment_loss‚4.733298563554827e-07 1loss/SRqqt1/detection_loss‚1.2943551399391786 1loss/SRqqt2/detection_loss‚1.0081356468368066 epoch‚0 trainer/global_step‚11099 %loss/total_loss‚16.02750418915676 /loss/FRt1/assignment_loss‚2.130275881909705 /loss/FRt1/detection_loss‚2.4946809770999017 _step‚887 /loss/FRt2/detection_loss‚2.3868139173756253 $ _timestamp‚1.785814912474068e+09 _runtime‚4657.663848045÷d}œ¸œí‚(:‘ ‡¿ÅÓ«Ì­ƒ gpu.0.gpu‚43.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚118.895' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚39.63166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0°Àcî‚(:Ø  ‡¿ÅÓŽ’⃠disk.sda.in‚ 11.6328125 network.recv‚647391 disk.drbd1673.out‚0$ disk.nvme0n1.out‚3963.3388671875 disk.drbd1356.in‚0 disk.drbd1550.in‚0 disk.sda2.in‚ 11.6328125 disk.sda2.out‚ 191.60546875 disk.drbd1356.out‚0 disk.drbd2041.in‚0 disk.drbd2111.out‚0 proc.cpu.threads‚29 disk.drbd1817.in‚0 disk.drbd1285.out‚0 disk.drbd1510.out‚0$ memory_percent‚41.24981164932251 disk.drbd1938.out‚0 disk.sda.out‚ 191.60546875 disk.drbd1226.in‚0 disk.drbd1510.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1938.in‚0 disk.drbd1931.in‚0 disk.drbd2111.in‚0) proc.memory.availableMB‚ 38502.5234375 proc.memory.rssMB‚11254 disk.drbd1673.in‚0 disk.rbd1.in‚0 disk.drbd1550.out‚0 disk.drbd1828.in‚0! disk.dm-0.out‚3963.3388671875 disk.drbd1552.out‚0 disk.drbd2118.out‚0 disk.drbd1289.in‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.drbd1931.out‚0 disk.drbd1828.out‚0 disk.drbd1552.in‚0 disk.drbd2041.out‚0 disk.drbd1226.out‚0 disk.drbd1965.in‚0 disk.drbd2078.out‚0) disk./.usagePercent‚32.96035208389768 disk.rbd0.out‚0 disk.drbd1285.in‚0 disk.rbd1.out‚15.33447265625 disk.drbd2118.in‚0 disk.drbd1965.out‚0 disk.dm-0.in‚440.0625( proc.memory.percent‚17.1722412109375 disk.nvme0n1.in‚440.0625 disk.drbd1584.in‚0$ disk./.usageGB‚589.3645706176758 disk.drbd2078.in‚0 network.sent‚ 1.0327009e+07 disk.rbd0.in‚0 cpu‚13.73276389667479 disk.drbd1584.out‚0#­¡2ï- _wandbruntime‚4677  _runtime‚4677öDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚201224667731456 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚191413933179904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 117741095 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚186657466045472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 191815122 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚11149 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74074029 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75843364 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201941156766208 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚191413933179904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9810734551552 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9786724238704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74074027 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9810821981184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10075980111360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚244 _runtime‚ 4677.67164724 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚196444190284176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 117750751 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9810821981184 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 120188416 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚186657483084832 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚201224667731456 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚191413933179904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 117741095 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚186657466045472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 191815122 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚11149 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9656 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74074029 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75843364 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201941156766208 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚191413933179904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9810734551552 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9786724238704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74074027 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9810821981184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10075980111360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚244 _runtime‚ 4677.67164724 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚196444190284176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 117750751 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9810821981184 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 120188416 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚186657483084832 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 117751618 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43903884 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚201959421583872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚186674206217792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9811701607936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚74080672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10077012908544 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10076875950080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚201242818590208 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201959259721728 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚196461910772314 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9787601711926 trainer/global_step‚11149 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 _runtime‚4678.017819553 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9811701607936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75849907 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9787687515162 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31946025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚74080672 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 117751618 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31946279 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚191882408675328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚75850163 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚191431116982272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 191832288 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚191431099942912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 117761288 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9811614171136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚890 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 191832288 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 191841960 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43903882 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚196461807929718 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 191841960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚201242818590208 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888n¯ÀÌè&÷âM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9811614171136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 2768240640 _step‚890 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 191832288 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 191841960 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43903882 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚196461807929718 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 191841960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚201242818590208 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74080670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚201242714114048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚191431099942912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 % _timestamp‚1.7858149328275177e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 117751618 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43903884 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚201959421583872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚186674206217792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9811701607936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚74080672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚24903680 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10077012908544 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10076875950080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚201242818590208 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201959259721728 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚196461910772314 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9787601711926 trainer/global_step‚11149 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚254 _runtime‚4678.017819553 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9811701607936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚256 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75849907 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9787687515162 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31946025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚74080672 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 117751618 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31946279 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚191882408675328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 2645753856 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚75850163 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 161862144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 136958464 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚191431116982272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 191832288 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚191431099942912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚85803236 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 117761288 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚86278144úÝïÏ2ø- _wandbruntime‚4678  _runtime‚4678sAŸ„÷ùñ &loss/total_loss‚15.261241649272524 epoch‚0 trainer/global_step‚11149 2loss/SRqqt1/assignment_loss‚0.8282957260351602 /loss/FRt2/detection_loss‚2.5562367347831074 0loss/SRqqt1/detection_loss‚0.993999148785064 % _timestamp‚1.7858149328287115e+09 _step‚891 4loss/FBt1/assignment_loss‚4.0054246682075244e-07 /loss/FBt2/detection_loss‚0.7488906886428595 .loss/FRt2/assignment_loss‚2.27114939418706 1loss/SRqqt2/detection_loss‚1.1888575185356396 _runtime‚4678.018501795 0loss/FRt1/assignment_loss‚2.0636459893800994 2loss/SRqqt2/assignment_loss‚1.0458282536494412 loss/FBt2/assignment_loss‚0 /loss/FRt1/detection_loss‚2.3341223199174483 /loss/FBt1/detection_loss‚1.2302154748141765 .~üúö /loss/FRt1/detection_loss‚2.3341223199174483 /loss/FBt1/detection_loss‚1.2302154748141765 &loss/total_loss‚15.261241649272524 epoch‚0 trainer/global_step‚11149 2loss/SRqqt1/assignment_loss‚0.8282957260351602 /loss/FRt2/detection_loss‚2.5562367347831074 0loss/SRqqt1/detection_loss‚0.993999148785064 % _timestamp‚1.7858149328287115e+09 _step‚891 4loss/FBt1/assignment_loss‚4.0054246682075244e-07 /loss/FBt2/detection_loss‚0.7488906886428595 .loss/FRt2/assignment_loss‚2.27114939418706 1loss/SRqqt2/detection_loss‚1.1888575185356396 _runtime‚4678.018501795 0loss/FRt1/assignment_loss‚2.0636459893800994 2loss/SRqqt2/assignment_loss‚1.0458282536494412 loss/FBt2/assignment_loss‚0ûBå½,œû‚(:‘ –¿ÅÓà³ð‚ gpu.0.gpu‚39.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚132.403' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.13433333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¥Ò2©^ü‚(:Ó  –¿ÅÓ—²¶ƒ disk.sda2.out‚ 191.859375 disk.rbd1.in‚0 disk.drbd1356.out‚0 disk.drbd1226.in‚0 disk.drbd1510.out‚0 disk.drbd1289.out‚0 disk.drbd1817.out‚0 disk.drbd1673.out‚0 disk.rbd1.out‚15.33447265625 proc.memory.rssMB‚11254 disk.sda.in‚ 11.6328125 disk.drbd2078.out‚0 disk.drbd2078.in‚0 disk.drbd2118.out‚0( proc.memory.percent‚17.1722412109375 disk.nvme0n1.in‚440.0625 disk.drbd1356.in‚0 disk.drbd1285.out‚0 disk.drbd2118.in‚0 disk.drbd1817.in‚0 disk.drbd1828.in‚0 disk.sda1.out‚0' proc.memory.availableMB‚ 38502.46875 disk.drbd1584.in‚0 disk.sda1.in‚0 disk.drbd1673.in‚0 disk.drbd1938.out‚0 disk.drbd1931.in‚0 disk.drbd1828.out‚0! disk.dm-0.out‚3963.5341796875$ disk.nvme0n1.out‚3963.5341796875 disk.drbd1584.out‚0 disk.drbd1226.out‚0 disk.drbd1550.out‚0 disk.drbd1285.in‚0 disk.drbd1552.out‚0* disk./.usagePercent‚32.960234961423055 disk.rbd0.out‚0 disk.drbd2041.out‚0 disk.dm-0.in‚440.0625 disk.drbd2111.in‚0 disk.drbd1289.in‚0 disk.drbd1552.in‚0 disk.drbd1938.in‚0 cpu‚13.73274141546091 proc.cpu.threads‚29 disk.drbd1965.out‚0 disk.sda2.in‚ 11.6328125 network.recv‚648794 disk.drbd1931.out‚0 disk.sda.out‚ 191.859375 disk.drbd1550.in‚0 disk.rbd0.in‚0% memory_percent‚41.249895095825195 network.sent‚ 1.0371953e+07# disk./.usageGB‚589.362476348877 disk.drbd2041.in‚0 disk.drbd1965.in‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0ËO‘ý‚(:† ¥¿ÅÓÍË™ƒ gpu.0.gpu‚42.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚133.299' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚44.433 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦@ºfþ‚(:Û  ¥¿ÅÓ§€Þƒ disk.drbd1965.in‚0 disk.drbd1673.out‚0 disk.rbd0.out‚0 disk.drbd1552.out‚0 disk.drbd1584.in‚0 disk.drbd1931.out‚0 disk.drbd2041.in‚0 disk.drbd1226.out‚0 proc.memory.rssMB‚11254 disk.drbd1673.in‚0 disk.sda2.in‚ 11.6328125" disk.dm-0.out‚3963.83447265625% disk.nvme0n1.out‚3963.83447265625 disk.drbd1510.out‚0 disk.drbd1552.in‚0 disk.drbd1938.out‚0 disk.rbd1.in‚0 network.recv‚650250 disk.drbd1828.out‚0 disk.drbd1289.out‚0 disk.drbd1938.in‚0 disk.sda.in‚ 11.6328125 network.sent‚ 1.0416922e+07 disk.sda2.out‚ 192.3828125 disk.rbd1.out‚15.36669921875 disk.nvme0n1.in‚440.0625* proc.memory.availableMB‚38502.45703125( proc.memory.percent‚17.1722412109375 disk.drbd2078.out‚0 disk.drbd2118.out‚0 disk.drbd1817.out‚0 disk.drbd1356.in‚0 disk.drbd1226.in‚0 disk.rbd0.in‚0 disk.drbd1817.in‚0 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.drbd1285.out‚0 disk.drbd1965.out‚0 disk.drbd1931.in‚0 cpu‚13.732557516947896 disk.drbd1828.in‚0 disk.drbd1510.in‚0$ memory_percent‚41.24991297721863 disk.drbd2111.out‚0 disk.sda1.out‚0 disk.drbd1285.in‚0* disk./.usagePercent‚32.960233894733854$ disk./.usageGB‚589.3624572753906 disk.drbd2078.in‚0 disk.dm-0.in‚440.0625 disk.drbd2111.in‚0 disk.drbd1550.out‚0 disk.sda1.in‚0 disk.drbd1289.in‚0 disk.drbd1550.in‚0 disk.drbd2118.in‚0 disk.drbd1584.out‚0 disk.sda.out‚ 192.3828125 proc.cpu.threads‚29Ð=ɘ2ÿ- _wandbruntime‚4698  _runtime‚4698ò÷ybȀ ' lr-AdamW/pg1‚0.00028862271644894297 ' lr-AdamW/pg2‚0.00028862271644894297 trainer/global_step‚11199 $ _timestamp‚1.785814953382575e+09 _runtime‚ 4698.57252441 _step‚892ÀÔŠÍÇ ' lr-AdamW/pg1‚0.00028862271644894297 ' lr-AdamW/pg2‚0.00028862271644894297 trainer/global_step‚11199 $ _timestamp‚1.785814953382575e+09 _runtime‚ 4698.57252441 _step‚892ü·VM+2‚- _wandbruntime‚4698  _runtime‚4698*ŒlEƒÐO VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚202127106486272 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 192684932 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 118278753 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74406179 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74406177 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚192272288369664 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9854730686976 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚202127106486272 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚192272271330304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚192725532494848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚238 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚202127002017280 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚192725557398528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚236 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚202846821033984 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚197325190206168 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9830698505336 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 192675274 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚74406177 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 118278753 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚76183327 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44096795 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚187494474661472dm÷˜Û'„ÕO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚202127002017280 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚192725557398528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚236 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚202846821033984 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚197325190206168 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9830698505336 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 192675274 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚74406177 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 118278753 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚76183327 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44096795 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚187494474661472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 2645753856 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚202127106486272 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 192684932 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 118278753 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74406179 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74406177 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚192272288369664 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9854730686976 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚202127106486272 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚192272271330304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚192725532494848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚238 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚18 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136ýL—'12…- _wandbruntime‚4698  _runtime‚4698@ñ¸ã&†ÝM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚192289455132672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚76190386 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚74412822 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚32089409 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9831490175434 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44100734 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 192702112 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74412820 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚187511231873152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10122039597056 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 192702112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚76190141 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 118279620 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚187511214833792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9855697743360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10122172361216 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚4698.919797101 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚192742764515328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚197342807851822 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚192289438093312 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚11199 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 192702112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚245 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚76190141 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 118279620 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚187511214833792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9855697743360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10122172361216 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 2768240640 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚121 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚130 _runtime‚4698.919797101 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚243 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚192742764515328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚197342807851822 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 2871874888 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚192289438093312 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚11199 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9831490175434 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44100734 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 192702112 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74412820 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚187511231873152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10122039597056 B ‰„ 2loss/SRqqt1/assignment_loss‚0.8502181987397441 3loss/FBt2/assignment_loss‚7.450580152834618e-09 .loss/FRt2/detection_loss‚2.610675306637076 1loss/SRqqt2/detection_loss‚1.1749531252069338 &loss/total_loss‚16.659750995410434 trainer/global_step‚11199 % _timestamp‚1.7858149537306697e+09 3loss/FBt1/assignment_loss‚1.966918270035478e-06 1loss/SRqqt1/detection_loss‚1.1145223872430745 /loss/FBt2/detection_loss‚1.3459727781591937 /loss/FRt1/assignment_loss‚2.305574011312772 _runtime‚4698.920440038 /loss/FRt2/assignment_loss‚2.391892307921301 2loss/SRqqt2/assignment_loss‚0.8614717028667068 /loss/FRt1/detection_loss‚2.5855788996774858 /loss/FBt1/detection_loss‚1.4188903032772942 epoch‚0 _step‚895‘8o¢Š‰ /loss/FRt2/assignment_loss‚2.391892307921301 2loss/SRqqt2/assignment_loss‚0.8614717028667068 /loss/FRt1/detection_loss‚2.5855788996774858 /loss/FBt1/detection_loss‚1.4188903032772942 epoch‚0 _step‚895 2loss/SRqqt1/assignment_loss‚0.8502181987397441 3loss/FBt2/assignment_loss‚7.450580152834618e-09 .loss/FRt2/detection_loss‚2.610675306637076 1loss/SRqqt2/detection_loss‚1.1749531252069338 &loss/total_loss‚16.659750995410434 trainer/global_step‚11199 % _timestamp‚1.7858149537306697e+09 3loss/FBt1/assignment_loss‚1.966918270035478e-06 1loss/SRqqt1/detection_loss‚1.1145223872430745 /loss/FBt2/detection_loss‚1.3459727781591937 /loss/FRt1/assignment_loss‚2.305574011312772 _runtime‚4698.920440038ÿÕmR ‘‹‚(:† ´¿ÅÓý¤¤ƒ gpu.0.gpu‚26.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚4.1587066650390625) gpu.0.memoryAllocatedBytes‚ 3572301824 gpu.0.temp‚39.0 gpu.0.powerWatts‚121.821' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚40.607 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0åñ©¼dŒ‚(:Ù  ´¿ÅÓÂÃǃ disk.drbd1226.in‚0 disk.drbd1285.out‚0 disk.drbd2111.in‚0 disk.drbd1510.out‚0 disk.drbd1552.in‚0" disk.dm-0.out‚3964.06884765625 disk.drbd1356.out‚0 disk.drbd2041.out‚0 disk.drbd1226.out‚0 disk.drbd2118.out‚0 disk.nvme0n1.in‚440.0625 disk.drbd2041.in‚0 disk.sda.out‚ 192.671875* disk./.usagePercent‚32.960234961423055 disk.drbd2118.in‚0% disk.nvme0n1.out‚3964.06884765625* proc.memory.availableMB‚38502.64453125( proc.memory.percent‚17.1722412109375 disk.drbd1285.in‚0 disk.drbd1828.in‚0 disk.sda1.out‚0 disk.drbd1673.in‚0 disk.rbd1.out‚15.36669921875 disk.drbd1584.in‚0 proc.memory.rssMB‚11254 cpu‚13.732351505517265 disk.drbd1552.out‚0 disk.drbd1931.out‚0 disk.drbd1828.out‚0 disk.drbd2111.out‚0 disk.drbd1510.in‚0 disk.drbd1289.in‚0 disk.sda2.out‚ 192.671875 disk.drbd2078.in‚0 disk.rbd0.in‚0 disk.drbd1550.out‚0 disk.drbd1938.in‚0 disk.sda2.in‚ 11.6328125 disk.rbd1.in‚0 disk.sda.in‚ 11.6328125# disk./.usageGB‚589.362476348877 disk.drbd1965.out‚0 disk.rbd0.out‚0 disk.drbd1817.out‚0 disk.drbd1584.out‚0 disk.drbd1673.out‚0 disk.drbd1938.out‚0 proc.cpu.threads‚29 disk.drbd1356.in‚0% memory_percent‚41.249626874923706 disk.drbd2078.out‚0 disk.drbd1289.out‚0 disk.sda1.in‚0 disk.drbd1817.in‚0 disk.drbd1931.in‚0 disk.drbd1965.in‚0 disk.dm-0.in‚440.0625 network.recv‚651405 disk.drbd1550.in‚0 network.sent‚ 1.0419987e+07 |ay2- _wandbruntime‚4719  _runtime‚4719QP{ÈŽÂ _runtime‚4719.192204861 _step‚896 & lr-AdamW/pg1‚0.0002899113257732083 & lr-AdamW/pg2‚0.0002899113257732083 trainer/global_step‚11249 % _timestamp‚1.7858149740022776e+09Ì°ÍÇ _step‚896 & lr-AdamW/pg1‚0.0002899113257732083 & lr-AdamW/pg2‚0.0002899113257732083 trainer/global_step‚11249 % _timestamp‚1.7858149740022776e+09 _runtime‚4719.192204861€ôT’Ÿ2- _wandbruntime‚4719  _runtime‚4719÷§TØ'‘ÒO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚188331500316832 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 118806755 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚74738327 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 193535426 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚193130626520064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 118806755 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44293647 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚242 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚193130626520064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚198206087285676 trainer/global_step‚11249 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74738329 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚203029336303104 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚193585888334848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10166469544448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9898726822400 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9874501165720 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚203752220913664 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9898814252032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚76523550 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚198205984443192 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 _runtime‚4719.196641651 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚203029336303104 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9898814252032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚203029336303104 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9898814252032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚548 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚74738327 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 193535426 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚193130626520064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 118806755 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44293647 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚242 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚193130626520064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 136965632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚198206087285676 trainer/global_step‚11249 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚86278144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74738329 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚85803124 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 2940455936 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚203029336303104 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚193585888334848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚24903680 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10166469544448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9898726822400 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9874501165720 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚203752220913664 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9898814252032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚76523550 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 104468992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚198205984443192 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 3021996032 _runtime‚4719.196641651 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9899606441984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚188348240489152 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 118807622 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚198223704931330 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 189288448 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚11249 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚193147793283072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32232851 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚203770490499072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9875378638942 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10167370143744 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 2712961024 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 193562264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 2940455936 LDDevice